From 7fd6a4b161522f9b3b07324ba6d829b047800bdc Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 16 Mar 2021 15:30:13 +0100 Subject: [PATCH 01/31] Dummy output: fix invalid counting of bytes and packets of biflow records --- src/plugins/output/dummy/dummy.c | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/plugins/output/dummy/dummy.c b/src/plugins/output/dummy/dummy.c index ed883dd5..f0079260 100644 --- a/src/plugins/output/dummy/dummy.c +++ b/src/plugins/output/dummy/dummy.c @@ -46,6 +46,11 @@ #include "config.h" +#define IANA_PEN 0 +#define IANA_PEN_REV 29305 +#define IE_ID_BYTES 1 +#define IE_ID_PKTS 2 + /** Plugin description */ IPX_API struct ipx_plugin_info ipx_plugin_info = { // Plugin type @@ -57,7 +62,7 @@ IPX_API struct ipx_plugin_info ipx_plugin_info = { // Configuration flags (reserved for future use) .flags = 0, // Plugin version string (like "1.2.3") - .version = "2.1.0", + .version = "2.2.0", // Minimal IPFIXcol version string (like "1.2.3") .ipx_min = "2.0.0" }; @@ -91,7 +96,8 @@ stats_update(struct instance_data *inst, ipx_msg_ipfix_t *msg) // For each IPFIX Data Record for (uint32_t i = 0; i < rec_cnt; ++i) { struct ipx_ipfix_record *rec_ptr = ipx_msg_ipfix_get_drec(msg, i); - enum fds_template_type ttype = rec_ptr->rec.tmplt->type; + const struct fds_template *tmplt = rec_ptr->rec.tmplt; + const enum fds_template_type ttype = tmplt->type; struct fds_drec_field field; uint64_t value; @@ -105,13 +111,30 @@ stats_update(struct instance_data *inst, ipx_msg_ipfix_t *msg) } // Get octetDeltaCount - if (fds_drec_find(&rec_ptr->rec, 0, 1, &field) != FDS_EOC + if (fds_drec_find(&rec_ptr->rec, IANA_PEN, IE_ID_BYTES, &field) != FDS_EOC && fds_get_uint_be(field.data, field.size, &value) == FDS_OK) { inst->cnt_bytes += value; } // Get packetDeltaCount - if (fds_drec_find(&rec_ptr->rec, 0, 2, &field) != FDS_EOC + if (fds_drec_find(&rec_ptr->rec, IANA_PEN, IE_ID_PKTS, &field) != FDS_EOC + && fds_get_uint_be(field.data, field.size, &value) == FDS_OK) { + inst->cnt_pkts += value; + } + + if ((tmplt->flags & FDS_TEMPLATE_BIFLOW) == 0) { + // Not a biflow record + continue; + } + + // Get octetDeltaCount (reverse) + if (fds_drec_find(&rec_ptr->rec, IANA_PEN_REV, IE_ID_BYTES, &field) != FDS_EOC + && fds_get_uint_be(field.data, field.size, &value) == FDS_OK) { + inst->cnt_bytes += value; + } + + // Get packetDeltaCount (reverse) + if (fds_drec_find(&rec_ptr->rec, IANA_PEN_REV, IE_ID_PKTS, &field) != FDS_EOC && fds_get_uint_be(field.data, field.size, &value) == FDS_OK) { inst->cnt_pkts += value; } From 5020ded49e410a0b961057f59e6269fc9c8c0195 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 16 Mar 2021 15:31:10 +0100 Subject: [PATCH 02/31] Bump version to 2.2.1 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e9d5e5ea..830a3795 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ endif() # Versions and other informations set(IPFIXCOL_VERSION_MAJOR 2) set(IPFIXCOL_VERSION_MINOR 2) -set(IPFIXCOL_VERSION_PATCH 0) +set(IPFIXCOL_VERSION_PATCH 1) set(IPFIXCOL_VERSION ${IPFIXCOL_VERSION_MAJOR}.${IPFIXCOL_VERSION_MINOR}.${IPFIXCOL_VERSION_PATCH}) From 39365b1769eb0b3047757aed9181d2490ee416e0 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 16 Mar 2021 16:45:16 +0100 Subject: [PATCH 03/31] DEB package: use latest collector version in changelog --- pkg/deb/CMakeLists.txt | 7 ++++++- pkg/deb/templates/{changelog => changelog.in} | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) rename pkg/deb/templates/{changelog => changelog.in} (53%) diff --git a/pkg/deb/CMakeLists.txt b/pkg/deb/CMakeLists.txt index 129af061..7e79bcf5 100644 --- a/pkg/deb/CMakeLists.txt +++ b/pkg/deb/CMakeLists.txt @@ -48,7 +48,6 @@ file(MAKE_DIRECTORY # Copy and create configuration files file(COPY - "templates/changelog" "templates/rules" "templates/compat" "templates/ipfixcol2.install" @@ -61,6 +60,12 @@ file(COPY DESTINATION "${DEB_CFG_DIR}/source" ) +configure_file( + "templates/changelog.in" + "${DEB_CFG_DIR}/changelog" + @ONLY +) + configure_file( "templates/control.in" "${DEB_CFG_DIR}/control" diff --git a/pkg/deb/templates/changelog b/pkg/deb/templates/changelog.in similarity index 53% rename from pkg/deb/templates/changelog rename to pkg/deb/templates/changelog.in index 603cd04b..2382e678 100644 --- a/pkg/deb/templates/changelog +++ b/pkg/deb/templates/changelog.in @@ -1,4 +1,4 @@ -ipfixcol2 (2.2.0-1) unstable; urgency=low +ipfixcol2 (@CPACK_PACKAGE_VERSION@-@CPACK_PACKAGE_RELEASE@) unstable; urgency=low * Initial release. From db9a5d4a56bd3433f4ff23935b731a60c0c6865c Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 16 Mar 2021 16:51:26 +0100 Subject: [PATCH 04/31] Unirec output: use required binary name instead of python2-docutils package in spec file --- extra_plugins/output/unirec/ipfixcol2-unirec-output.spec.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra_plugins/output/unirec/ipfixcol2-unirec-output.spec.in b/extra_plugins/output/unirec/ipfixcol2-unirec-output.spec.in index 71c348de..0f8bd1d0 100644 --- a/extra_plugins/output/unirec/ipfixcol2-unirec-output.spec.in +++ b/extra_plugins/output/unirec/ipfixcol2-unirec-output.spec.in @@ -12,7 +12,7 @@ Packager: BuildRoot: %{_tmppath}/%{name}-%{version}-%{release} BuildRequires: gcc >= 4.8, cmake >= 2.8.8, make -BuildRequires: ipfixcol2-devel, libfds-devel, python2-docutils +BuildRequires: ipfixcol2-devel, libfds-devel, /usr/bin/rst2man BuildRequires: libtrap-devel, unirec >= 2.3.0 Requires: libtrap >= 0.12.0, ipfixcol2 >= 2.0.0, libfds >= 0.1.0 From ac9689df56448a25b05d0055b87393f9d055b5cb Mon Sep 17 00:00:00 2001 From: Jiri Havranek Date: Thu, 25 Mar 2021 21:03:11 +0100 Subject: [PATCH 05/31] Unirec output: fixed segfault when duplicated field mapping is present --- extra_plugins/output/unirec/src/map.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extra_plugins/output/unirec/src/map.c b/extra_plugins/output/unirec/src/map.c index 415facda..8c3b8b6f 100644 --- a/extra_plugins/output/unirec/src/map.c +++ b/extra_plugins/output/unirec/src/map.c @@ -480,9 +480,9 @@ map_sort_fn(const void *p1, const void *p2) ipfix2 = ipfix2->next; } - if (ipfix1->next) { + if (ipfix1 && ipfix1->next) { return -1; - } else if (ipfix2->next) { + } else if (ipfix2 && ipfix2->next) { return 1; } @@ -585,7 +585,7 @@ map_load(map_t *map, const char *file) // Collision detected! snprintf(map->err_buffer, ERR_SIZE, "The IPFIX IE '%s' (PEN %" PRIu32 ", ID %" PRIu16 ") " - "is mapped to multiple different UniRec fields ('%s' and '%s')", + "is mapped to multiple UniRec fields ('%s' and '%s')", rec_now->ipfix.def->name, rec_now->ipfix.en, rec_now->ipfix.id, rec_now->unirec.name, rec_prev->unirec.name); map_clear(map); From f552aa79e6b208b894ef0365652cabd30f26b092 Mon Sep 17 00:00:00 2001 From: Jiri Havranek Date: Fri, 26 Mar 2021 15:47:40 +0100 Subject: [PATCH 06/31] Unirec output: fixed unknown parameter error on startup --- extra_plugins/output/unirec/src/configuration.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/extra_plugins/output/unirec/src/configuration.c b/extra_plugins/output/unirec/src/configuration.c index 5d8a3c02..bc40455e 100644 --- a/extra_plugins/output/unirec/src/configuration.c +++ b/extra_plugins/output/unirec/src/configuration.c @@ -480,9 +480,10 @@ cfg_parse_tcp(ipx_ctx_t *ctx, fds_xml_ctx_t *root, struct conf_params *cfg, enum // Prepare the TRAP interface specification char *res = NULL; const char *trap_ifc = (type == SPEC_TCP) ? "t" : "T"; - if (cfg_str_append(&res, "%s:%" PRIu16 ":%" PRIu64, trap_ifc, port, max_conn) != IPX_OK + if (cfg_str_append(&res, "%s:%" PRIu16, trap_ifc, port) != IPX_OK || (type == SPEC_TCP_TLS - && cfg_str_append(&res, ":%s:%s:%s", file_key, file_cert, file_ca) != IPX_OK)) { + && cfg_str_append(&res, ":%s:%s:%s", file_key, file_cert, file_ca) != IPX_OK) + || (cfg_str_append(&res, ":max_clients=%" PRIu64, max_conn) != IPX_OK)) { IPX_CTX_ERROR(ctx, "Unable to allocate memory (%s:%d)", __FILE__, __LINE__); free(file_ca); free(file_cert); free(file_key); free(res); @@ -544,7 +545,7 @@ cfg_parse_unix(ipx_ctx_t *ctx, fds_xml_ctx_t *root, struct conf_params *cfg) // Prepare the TRAP interface specification char *res = NULL; - if (cfg_str_append(&res, "u:%s:%" PRIu64, name, max_conn) != IPX_OK) { + if (cfg_str_append(&res, "u:%s:max_clients=%" PRIu64, name, max_conn) != IPX_OK) { IPX_CTX_ERROR(ctx, "Unable to allocate memory (%s:%d)", __FILE__, __LINE__); free(name); return IPX_ERR_NOMEM; From b13b9e31d2a988d67ad5130630cce7af2055510b Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Fri, 16 Apr 2021 13:50:35 +0200 Subject: [PATCH 07/31] Unirec output: add conversion from iana:icmpTypeCodeIPv4 to UniRec DST_PORT Destination port is often used to store ICMP type/code in nfdump and other tools, therefore, it is convenient to use this conversion for sake of simplicity. --- extra_plugins/output/unirec/config/unirec-elements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra_plugins/output/unirec/config/unirec-elements.txt b/extra_plugins/output/unirec/config/unirec-elements.txt index 74b18fdd..75612126 100644 --- a/extra_plugins/output/unirec/config/unirec-elements.txt +++ b/extra_plugins/output/unirec/config/unirec-elements.txt @@ -24,7 +24,7 @@ SRC_IP ipaddr e0id8,e0id27 # IPv4 or IPv6 source address DST_IP ipaddr e0id12,e0id28 # IPv4 or IPv6 destination address SRC_PORT uint16 e0id7 # Transport protocol source port -DST_PORT uint16 e0id11 # Transport protocol destination port +DST_PORT uint16 e0id11,e0id32 # Transport protocol destination port or ICMP type/code PROTOCOL uint8 e0id4 # Transport protocol TCP_FLAGS uint8 e0id6 # TCP flags BYTES uint64 e0id1 # Number of bytes in flow From d392cc91a0ff4d3ebec326f8d28bb1f5ef2decff Mon Sep 17 00:00:00 2001 From: Petr Velan Date: Thu, 20 May 2021 18:31:30 +0200 Subject: [PATCH 08/31] Fixed duplicit record output before template entries in JSON plugin --- src/plugins/output/json/src/Storage.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plugins/output/json/src/Storage.cpp b/src/plugins/output/json/src/Storage.cpp index 498d37f5..47ebad5c 100644 --- a/src/plugins/output/json/src/Storage.cpp +++ b/src/plugins/output/json/src/Storage.cpp @@ -276,6 +276,9 @@ Storage::convert_tset(struct ipx_ipfix_set *set, const struct fds_ipfix_msg_hdr // Iteration through all (Options) Templates in the Set while (fds_tset_iter_next(&tset_iter) == FDS_OK) { + // Buffer is empty + m_record.size_used = 0; + // Read and print single template convert_tmplt_rec(&tset_iter, set_id, hdr); @@ -285,9 +288,6 @@ Storage::convert_tset(struct ipx_ipfix_set *set, const struct fds_ipfix_msg_hdr return IPX_ERR_DENIED; } } - - // Buffer is empty - m_record.size_used = 0; } return IPX_OK; @@ -453,4 +453,4 @@ Storage::convert(struct fds_drec &rec, const fds_iemgr_t *iemgr, fds_ipfix_msg_h // Append the record with end of line character buffer_append("\n"); -} \ No newline at end of file +} From d0b6e8939876089ef2077680a222191ff192a4c3 Mon Sep 17 00:00:00 2001 From: Tomas Cejka Date: Tue, 20 Jul 2021 16:47:51 +0200 Subject: [PATCH 09/31] Unirec output: cleaned and remapped HTTP fields Removed obsoleted IPFIX elements from older flowmonexp plugin with MUNI enterprise ID, renamed UniRec fields to match the names in ipfixprobe and NEMEA. --- .../output/unirec/config/unirec-elements.txt | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/extra_plugins/output/unirec/config/unirec-elements.txt b/extra_plugins/output/unirec/config/unirec-elements.txt index 75612126..949fa623 100644 --- a/extra_plugins/output/unirec/config/unirec-elements.txt +++ b/extra_plugins/output/unirec/config/unirec-elements.txt @@ -118,15 +118,6 @@ FME_SIP_CALLED_PARTY string flowmon:sipCalledParty FME_SIP_VIA string flowmon:sipVia # SIP VIA # --- HTTP elements --- -HTTP_REQUEST_METHOD_ID uint32 e16982id500 # HTTP request method id -HTTP_REQUEST_HOST string e16982id501 # HTTP(S) request host -HTTP_REQUEST_URL string e16982id502 # HTTP request url -HTTP_REQUEST_AGENT_ID uint32 e16982id503 # HTTP request agent id -HTTP_REQUEST_AGENT string e16982id504 # HTTP request agent -HTTP_REQUEST_REFERER string e16982id505 # HTTP referer -HTTP_RESPONSE_STATUS_CODE uint32 e16982id506 # HTTP response status code -HTTP_RESPONSE_CONTENT_TYPE string e16982id507 # HTTP response content type - FME_HTTP_UA_OS uint16 flowmon:httpUaOs FME_HTTP_UA_OS_MAJ uint16 flowmon:httpUaOsMaj FME_HTTP_UA_OS_MIN uint16 flowmon:httpUaOsMin @@ -136,10 +127,14 @@ FME_HTTP_UA_APP_MAJ uint16 flowmon:httpUaAppMaj FME_HTTP_UA_APP_MIN uint16 flowmon:httpUaAppMin FME_HTTP_UA_APP_BLD uint16 flowmon:httpUaAppBld FME_HTTP_METHOD_MASK uint16 flowmon:httpMethodMask -FME_HTTP_REQUEST_HOST string flowmon:httpHost # HTTP(S) request host -FME_HTTP_REQUEST_URL string flowmon:httpUrl # HTTP request url -FME_HTTP_RESPONSE_STATUS_CODE uint32 flowmon:httpStatusCode # HTTP response status code -FME_HTTP_REQUEST_USER_AGENT string flowmon:httpUserAgent + +HTTP_REQUEST_AGENT string flowmon:httpUserAgent +HTTP_REQUEST_HOST string flowmon:httpHost # HTTP(S) request host +HTTP_REQUEST_REFERER string flowmon:httpReferer +HTTP_REQUEST_URL string flowmon:httpUrl # HTTP request url +HTTP_REQUEST_METHOD string cesnet:httpMethod # HTTP method text representation +HTTP_RESPONSE_STATUS_CODE uint16 flowmon:httpStatusCode # HTTP response status code +HTTP_RESPONSE_CONTENT_TYPE string flowmon:httpContentType # HTTP ContentType text representation # --- Other fields --- IPV6_TUN_TYPE uint8 e16982id405 # IPv6 tunnel type From 36b1e9a9492ce4b3d3b30e439b85c6e72b6ee48e Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Thu, 7 Oct 2021 14:54:02 +0200 Subject: [PATCH 10/31] TCP input: replace waiting for whole IPFIX Messages with continous buffering of its parts The fixes situation when a connection with an exporter is unexpectly closed by the collector when a part of IPFIX message is sent over TCP but the rest is still on the way for long pariod of time. Previously the plugin waited up to half a second for the rest of the message. However, if no parts have been received, the connection was prematurely closed due to connection timeout. Moreover, waiting for IPFIX Message parts (i.e. blocking) could caused performance degredation if multiple exporters were connected to the collector at the same time. In this commit, input sockets of all expoters are always non-blocking and parts of IPFIX Messages are buffered until the whole IPFIX Message is received. --- src/plugins/input/tcp/tcp.c | 294 +++++++++++++++++++++++++++++------- 1 file changed, 237 insertions(+), 57 deletions(-) diff --git a/src/plugins/input/tcp/tcp.c b/src/plugins/input/tcp/tcp.c index e501804f..84b40083 100644 --- a/src/plugins/input/tcp/tcp.c +++ b/src/plugins/input/tcp/tcp.c @@ -47,6 +47,7 @@ #include #include #include +#include #include #include @@ -62,10 +63,6 @@ #define GETTER_TIMEOUT (10) /** Max sockets events processed in the getter - i.e. epoll_wait array size */ #define GETTER_MAX_EVENTS (16) -/** Timeout to read whole IPFIX Message after at least part has been received (in microseconds) */ -#define GETTER_RECV_TIMEOUT (500000) -/** Default size of a buffer prepared for new IPFIX/NetFlow message (bytes) */ -#define DEF_MSG_SIZE (4096) /** Plugin description */ IPX_API struct ipx_plugin_info ipx_plugin_info = { @@ -91,6 +88,13 @@ struct tcp_pair { struct ipx_session *session; /** No message has been received from the Session yet */ bool new_connection; + + /** Partly received message that waits for completition */ + uint8_t *msg; + /** Allocated size of the partly received msg message */ + uint16_t msg_size; + /** Already receive part of the msg message */ + uint16_t msg_offset; }; /** Instance data */ @@ -143,7 +147,7 @@ static int active_session_add(struct tcp_data *data, int sd, struct ipx_session *session) { // Create a new pair - struct tcp_pair *pair = malloc(sizeof(*pair)); + struct tcp_pair *pair = calloc(1, sizeof(*pair)); if (!pair) { IPX_CTX_ERROR(data->ctx, "Memory allocation failed! (%s:%d)", __FILE__, __LINE__); return IPX_ERR_NOMEM; @@ -242,7 +246,11 @@ active_session_remove_aux(struct tcp_data *data, size_t idx) } } - // Close internal structures an remove it from the list (do NOT free SESSION) + // Free internal structures and remove the pair from the list (do NOT free SESSION) + if (pair->msg) { + free(pair->msg); + } + close(pair->fd); free(pair); @@ -309,14 +317,21 @@ listener_add_connection(struct tcp_data *data, int sd) assert(sd >= 0); const char *err_str; - // Set receive timeout (after data on the socket is ready) - struct timeval rcv_timeout; - rcv_timeout.tv_sec = GETTER_RECV_TIMEOUT / 1000000; - rcv_timeout.tv_usec = GETTER_RECV_TIMEOUT % 1000000; - if (setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &rcv_timeout, sizeof(rcv_timeout)) == -1) { + // Set non-blocking mode on the socket + int flags = fcntl(sd, F_GETFL, 0); + if (flags == -1) { ipx_strerror(errno, err_str); - IPX_CTX_WARNING(data->ctx, "Listener: Failed to specify receiving timeout of a socket: %s", + IPX_CTX_WARNING(data->ctx, "Listener: Failed to set non-blocking mode: fcntl() failed: %s", err_str); + return IPX_ERR_DENIED; + } + + flags |= O_NONBLOCK; + if (fcntl(sd, F_SETFL, flags) == -1) { + ipx_strerror(errno, err_str); + IPX_CTX_WARNING(data->ctx, "Listener: Failed to set non-blocking mode: fcntl() failed: %s", + err_str); + return IPX_ERR_DENIED; } // Get the description of the remove address @@ -810,102 +825,267 @@ active_destroy(ipx_ctx_t *ctx, struct tcp_data *instance) } /** - * \brief Get an IPFIX message from a socket and pass it + * \brief Try to read an IPFIX Message header + * + * The whole header might not be available right now. In that case, the function + * will only read and store available part and it will successfully return. When + * this happens, the allocated message is smaller that IPFIX Message header. The + * next call of this function will tried to read the rest. + * + * If the whole header is available, the function will allocate message buffer + * sufficient for the rest of the message. * * \param[in] ctx Instance data (necessary for passing messages) * \param[in] pair Connection pair (socket descriptor and session) to receive from * \return #IPX_OK on success - * \return #IPX_ERR_EOF if the socket is closed + * \return #IPX_ERR_EOF if the socket has been closed * \return #IPX_ERR_FORMAT if the message (or stream) is malformed and the connection MUST be closed * \return #IPX_ERR_NOMEM on a memory allocation error and the connection MUST be closed */ static int -socket_process(ipx_ctx_t *ctx, struct tcp_pair *pair) +socket_process_receive_header(ipx_ctx_t *ctx, struct tcp_pair *pair) { - const char *err_str; - struct fds_ipfix_msg_hdr hdr; - static_assert(sizeof(hdr) == FDS_IPFIX_MSG_HDR_LEN, "Invalid size of IPFIX Message header"); + struct fds_ipfix_msg_hdr hdr = {0}; + uint8_t *hdr_raw = (uint8_t *) &hdr; - // Get the message header (do not move pointer) - ssize_t len = recv(pair->fd, &hdr, FDS_IPFIX_MSG_HDR_LEN, MSG_WAITALL | MSG_PEEK); + uint16_t remains = FDS_IPFIX_MSG_HDR_LEN; + uint16_t offset = 0; + ssize_t len; + + uint8_t *msg_buffer; + uint16_t msg_version; + uint16_t msg_size; + + assert(!pair->msg || pair->msg_offset < FDS_IPFIX_MSG_HDR_LEN); + + if (pair->msg) { + // Fill the header with previously received data + offset = pair->msg_offset; + remains = FDS_IPFIX_MSG_HDR_LEN - offset; + + memcpy(hdr_raw, pair->msg, offset); + } + + len = recv(pair->fd, &hdr_raw[offset], remains, 0); if (len == 0) { - // Connection terminated - IPX_CTX_INFO(ctx, "Connection from '%s' closed.", pair->session->ident); - return IPX_ERR_EOF; + // Connection has been closed + if (offset > 0) { + IPX_CTX_WARNING(ctx, "Connection with '%s' has been unexpectly closed", + pair->session->ident); + return IPX_ERR_FORMAT; + } else { + IPX_CTX_INFO(ctx, "Connection with '%s' closed.", pair->session->ident); + return IPX_ERR_EOF; + } } - if (len == -1 || len < FDS_IPFIX_MSG_HDR_LEN) { - // Failed to read header -> close - int error_code = (len == -1) ? errno : EINTR; - ipx_strerror(error_code, err_str); - IPX_CTX_WARNING(ctx, "Connection from '%s' closed due to failure to receive " - "an IPFIX Message header: %s", pair->session->ident, err_str); + if (len < 0) { + // Something went wrong + const char *err_str; + + if (errno == EWOULDBLOCK || errno == EAGAIN) { + return IPX_OK; + } + + ipx_strerror(errno, err_str); + IPX_CTX_WARNING(ctx, "Connection with '%s' failed: %s", + pair->session->ident, err_str); return IPX_ERR_FORMAT; } - assert(len == FDS_IPFIX_MSG_HDR_LEN); - // Check the header (version, size) - uint16_t msg_version = ntohs(hdr.version); - uint16_t msg_size = ntohs(hdr.length); - uint32_t msg_odid = ntohl(hdr.odid); + offset += len; + + if (offset < FDS_IPFIX_MSG_HDR_LEN) { + // We don't have whole IPFIX Message header + uint8_t *ptr = realloc(pair->msg, offset); + if (!ptr) { + IPX_CTX_ERROR(ctx, + "Connection with '%s' closed due to memory allocation failure! (%s:%d).", + pair->session->ident, __FILE__, __LINE__); + return IPX_ERR_NOMEM; + } + + memcpy(ptr, hdr_raw, offset); + pair->msg = ptr; + pair->msg_offset = offset; + pair->msg_size = offset; + + return IPX_OK; + } + + // Check the IPFIX Message header + msg_version = ntohs(hdr.version); + msg_size = ntohs(hdr.length); if (msg_version != FDS_IPFIX_VERSION || msg_size < FDS_IPFIX_MSG_HDR_LEN) { - // Unsupported header version - IPX_CTX_WARNING(ctx, "Connection from '%s' closed due to the unsupported version of " - "IPFIX/NetFlow.", pair->session->ident); + // Invalid header version + IPX_CTX_WARNING(ctx, + "Connection with '%s' closed due to invalid IPFIX Message header.", + pair->session->ident); return IPX_ERR_FORMAT; } - // Read the whole message - uint8_t *buffer = malloc(msg_size * sizeof(uint8_t)); - if (!buffer) { - IPX_CTX_ERROR(ctx, "Connection from '%s' closed due to memory allocation failure! (%s:%d).", + // Preallocated buffer for the rest of the IPFIX Message body + msg_buffer = realloc(pair->msg, msg_size); + if (!msg_buffer) { + IPX_CTX_ERROR(ctx, + "Connection with '%s' closed due to memory allocation failure! (%s:%d).", pair->session->ident, __FILE__, __LINE__); return IPX_ERR_NOMEM; } - len = recv(pair->fd, buffer, msg_size, MSG_WAITALL); - if (len != msg_size) { - int error_code = (len == -1) ? errno : ETIMEDOUT; - ipx_strerror(error_code, err_str); - IPX_CTX_ERROR(ctx, "Connection from '%s' closed due to failure while reading from " - "its socket: %s.", pair->session->ident, err_str); - free(buffer); + memcpy(msg_buffer, hdr_raw, FDS_IPFIX_MSG_HDR_LEN); + pair->msg = msg_buffer; + pair->msg_offset = FDS_IPFIX_MSG_HDR_LEN; + pair->msg_size = msg_size; + + return IPX_OK; +} + +/** + * \brief Try to read an IPFIX Message body + * + * The whole body might not be available right now. In that case, the function + * will only read and store available part and it will successfully return. When + * this happens, the message offset is smaller that the message size. The next call + * of this function will tried to read the rest. + * + * \param[in] ctx Instance data (necessary for passing messages) + * \param[in] pair Connection pair (socket descriptor and session) to receive from + * \return #IPX_OK on success + * \return #IPX_ERR_FORMAT if the message (or stream) is malformed and the connection MUST be closed + * \return #IPX_ERR_NOMEM on a memory allocation error and the connection MUST be closed + */ +static int +socket_process_receive_body(ipx_ctx_t *ctx, struct tcp_pair *pair) +{ + const uint16_t remains = pair->msg_size - pair->msg_offset; + assert(pair->msg && pair->msg_offset >= FDS_IPFIX_MSG_HDR_LEN); + + if (remains == 0) { + // This is an IPFIX Message without body... + return IPX_OK; + } + + ssize_t len = recv(pair->fd, &pair->msg[pair->msg_offset], remains, 0); + if (len == 0) { + // Connection closed + IPX_CTX_WARNING(ctx, "Connection from '%s' has been unexpectly closed", + pair->session->ident); return IPX_ERR_FORMAT; } + if (len < 0) { + // Something went wrong + const char *err_str; + + if (errno == EWOULDBLOCK || errno == EAGAIN) { + return IPX_OK; + } + + ipx_strerror(errno, err_str); + IPX_CTX_WARNING(ctx, "Connection with '%s' failed: %s", + pair->session->ident, err_str); + return IPX_ERR_FORMAT; + } + + pair->msg_offset += (uint16_t) len; + return IPX_OK; +} + +/** + * \brief Try to pass fully received IPFIX Message to the collector. + * + * \param[in] ctx Instance data (necessary for passing messages) + * \param[in] pair Connection pair (socket descriptor and session) + * \return #IPX_OK on success + * \return #IPX_ERR_NOMEM on a memory allocation error and the connection MUST be closed + */ +static int +socket_process_pass_msg(ipx_ctx_t *ctx, struct tcp_pair *pair) +{ + assert(pair->msg && pair->msg_offset == pair->msg_size && "Partly received message"); + if (pair->new_connection) { // Send information about the new Transport Session - pair->new_connection = false; ipx_msg_session_t *msg = ipx_msg_session_create(pair->session, IPX_MSG_SESSION_OPEN); if (!msg) { - IPX_CTX_ERROR(ctx, "Connection from '%s' closed due to memory allocation " - "failure! (%s:%d).", pair->session->ident, __FILE__, __LINE__); - free(buffer); + IPX_CTX_ERROR(ctx, + "Connection with '%s' closed due to memory allocation failure! (%s:%d).", + pair->session->ident, __FILE__, __LINE__); return IPX_ERR_NOMEM; } ipx_ctx_msg_pass(ctx, ipx_msg_session2base(msg)); + pair->new_connection = false; } // Create a message wrapper and pass the message struct ipx_msg_ctx msg_ctx; msg_ctx.session = pair->session; - msg_ctx.odid = msg_odid; + msg_ctx.odid = ntohl(((struct fds_ipfix_msg_hdr *) pair->msg)->odid); msg_ctx.stream = 0; // Streams are not supported over TCP - ipx_msg_ipfix_t *msg = ipx_msg_ipfix_create(ctx, &msg_ctx, buffer, msg_size); + ipx_msg_ipfix_t *msg = ipx_msg_ipfix_create(ctx, &msg_ctx, pair->msg, pair->msg_size); if (!msg) { - IPX_CTX_ERROR(ctx, "Connection from '%s' closed due to memory allocation " - "failure! (%s:%d).", pair->session->ident, __FILE__, __LINE__); - free(buffer); + IPX_CTX_ERROR(ctx, + "Connection with '%s' closed due to memory allocation failure! (%s:%d).", + pair->session->ident, __FILE__, __LINE__); return IPX_ERR_NOMEM; } ipx_ctx_msg_pass(ctx, ipx_msg_ipfix2base(msg)); + + pair->msg = NULL; + pair->msg_offset = 0; + pair->msg_size = 0; + return IPX_OK; } +/** + * \brief Get an IPFIX message from a socket and pass it + * + * \param[in] ctx Instance data (necessary for passing messages) + * \param[in] pair Connection pair (socket descriptor and session) to receive from + * \return #IPX_OK on success + * \return #IPX_ERR_EOF if the socket is closed + * \return #IPX_ERR_FORMAT if the message (or stream) is malformed and the connection MUST be closed + * \return #IPX_ERR_NOMEM on a memory allocation error and the connection MUST be closed + */ +static int +socket_process(ipx_ctx_t *ctx, struct tcp_pair *pair) +{ + int ret; + + if (!pair->msg || pair->msg_offset < FDS_IPFIX_MSG_HDR_LEN) { + // Try to receive IPFIX Message header first + ret = socket_process_receive_header(ctx, pair); + if (ret != IPX_OK) { + return ret; + } + + if (pair->msg_offset < FDS_IPFIX_MSG_HDR_LEN) { + // Incomplete IPFIX Message header, read the rest later... + return IPX_OK; + } + } + + // Receive rest of the message body + ret = socket_process_receive_body(ctx, pair); + if (ret != IPX_OK) { + return ret; + } + + if (pair->msg_offset < pair->msg_size) { + // Incomplete IPFIX Message body, read the rest later... + return IPX_OK; + } + + // Pass the message + return socket_process_pass_msg(ctx, pair); +} + // ------------------------------------------------------------------------------------------------- int From c8e770579fbf276bc92337ccc13a87257789db32 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Mon, 18 Oct 2021 10:25:00 +0200 Subject: [PATCH 11/31] JSON output: fix missing stdexcept Reported-by: madskills700 and thorgrin --- src/plugins/output/json/src/Config.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/output/json/src/Config.cpp b/src/plugins/output/json/src/Config.cpp index 673a3093..0414fb40 100644 --- a/src/plugins/output/json/src/Config.cpp +++ b/src/plugins/output/json/src/Config.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include From 6b2720ecaf98751e2628715c5bcf66f3b8e88f07 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Mon, 18 Oct 2021 10:27:17 +0200 Subject: [PATCH 12/31] JSON-Kafka output: fix missing stdexcept Reported-by: madskills700 and thorgrin --- src/plugins/output/json-kafka/src/Config.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/output/json-kafka/src/Config.cpp b/src/plugins/output/json-kafka/src/Config.cpp index 50f51342..42a021b8 100644 --- a/src/plugins/output/json-kafka/src/Config.cpp +++ b/src/plugins/output/json-kafka/src/Config.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include From 7de8d29341b8f36b5b32b83b1f669950f4369b94 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Fri, 3 Dec 2021 23:22:13 +0100 Subject: [PATCH 13/31] Core: pushlish ipx_msg_ipfix_add_{set,drec}_ref() functions --- include/ipfixcol2/message_ipfix.h | 24 ++++++++++++++++++++++++ src/core/message_ipfix.h | 22 ---------------------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/include/ipfixcol2/message_ipfix.h b/include/ipfixcol2/message_ipfix.h index c45c1941..a0bb1f92 100644 --- a/include/ipfixcol2/message_ipfix.h +++ b/include/ipfixcol2/message_ipfix.h @@ -201,6 +201,30 @@ ipx_msg_ipfix2base(ipx_msg_ipfix_t *msg) return (ipx_msg_t *) msg; } +/** + * \brief Add a new IPFIX Set description. + * + * The record is uninitialized and user MUST fill it! The function is + * intended for annotation of newly created IPFIX Message. + * \param[in] msg IPFIX Message wrapper + * \return Pointer to the record or NULL (memory allocation error) + */ +IPX_API struct ipx_ipfix_set * +ipx_msg_ipfix_add_set_ref(struct ipx_msg_ipfix *msg); + +/** + * \brief Add a new IPFIX Data Record description. + * + * The record is uninitialized and user MUST fill it! The function is + * intended for annotation of newly created IPFIX Message. + * \warning The wrapper \p msg_ref can be reallocated and different pointer + * can be returned! + * \param[in,out] msg_ref IPFIX Message wrapper + * \return Pointer to the record or NULL (memory allocation error) + */ +IPX_API struct ipx_ipfix_record * +ipx_msg_ipfix_add_drec_ref(struct ipx_msg_ipfix **msg_ref); + /**@}*/ #ifdef __cplusplus } diff --git a/src/core/message_ipfix.h b/src/core/message_ipfix.h index 3719a6cb..60fed909 100644 --- a/src/core/message_ipfix.h +++ b/src/core/message_ipfix.h @@ -116,26 +116,4 @@ struct ipx_msg_ipfix { size_t ipx_msg_ipfix_size(uint32_t rec_cnt, size_t rec_size); -/** - * \brief Add a new IPFIX Set - * - * The record is uninitialized and user MUST fill it! - * \param[in] msg IPFIX Message wrapper - * \return Pointer to the record or NULL (memory allocation error) - */ -struct ipx_ipfix_set * -ipx_msg_ipfix_add_set_ref(struct ipx_msg_ipfix *msg); - -/** - * \brief Add a new IPFIX Data Record - * - * The record is uninitialized and user MUST fill it! - * \warning The wrapper \p msg_ref can be reallocated and different pointer can be returned! - * \param[in,out] msg_ref IPFIX Message wrapper - * \return Pointer to the record or NULL (memory allocation error) - */ -struct ipx_ipfix_record * -ipx_msg_ipfix_add_drec_ref(struct ipx_msg_ipfix **msg_ref); - - #endif // IPFIXCOL_MESSAGE_IPFIX_INTERNAL_H From a05bd4cee4bf2477117ba157f98e79a470e5a349 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 8 Feb 2022 14:53:07 +0100 Subject: [PATCH 14/31] CI: replace actions/checkout@v1 with actions/checkout@v2 --- .github/workflows/main.yml | 2 +- .github/workflows/packages.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dee6c5a2..09ca96fe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,7 +20,7 @@ jobs: name: Build on ${{ matrix.image }} container: ${{ matrix.image }} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 # Dependencies --------------------------------------------------------------------------- - name: Install dependencies for libfds and IPFIXcol2 (Ubuntu/Debian) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 30170954..2f7ac6ea 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -22,7 +22,7 @@ jobs: container: ${{ matrix.image }} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Define global variables run: echo "::set-output name=zip_file::ipfixcol2-${IMAGE//:/}-$GITHUB_SHA.zip" shell: bash @@ -91,7 +91,7 @@ jobs: container: ${{ matrix.image }} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Prepare environment and variables run: | echo "::set-output name=zip_file::ipfixcol2-${IMAGE//:/}-$GITHUB_SHA.zip" From 4d75aba145bb5c5a9d5e3bef55c8f8766934aeb1 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 8 Feb 2022 15:29:33 +0100 Subject: [PATCH 15/31] CI: utilize upload-artifact@v2 for artifact archivation --- .github/workflows/packages.yml | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 2f7ac6ea..7d70fa95 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -67,17 +67,16 @@ jobs: ipfixcol2 -V ipfixcol2 -h ipfixcol2 -L - - name: Pack IPFIXcol2 DEB packages - working-directory: 'build/pkg/deb/debbuild/' - run: zip "$GITHUB_WORKSPACE/$ZIP_FILE" *.deb *.ddeb *.tar.gz *.dsc - env: - ZIP_FILE: ${{ steps.vars.outputs.zip_file }} - name: Archive DEB packages if: github.event_name == 'push' - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: ${{ steps.vars.outputs.zip_file }} - path: ${{ steps.vars.outputs.zip_file }} + path: | + build/pkg/deb/debbuild/*.deb + build/pkg/deb/debbuild/*.ddeb + build/pkg/deb/debbuild/*.tar.gz + build/pkg/deb/debbuild/*.dsc rpm: # Try to build RPM packages @@ -146,14 +145,11 @@ jobs: ipfixcol2 -V ipfixcol2 -h ipfixcol2 -L -v - - name: Pack IPFIXcol2 RPM packages - working-directory: 'build/pkg/rpm/rpmbuild' - run: zip -r "$GITHUB_WORKSPACE/$ZIP_FILE" RPMS SRPMS - env: - ZIP_FILE: ${{ steps.vars.outputs.zip_file }} - name: Archive RPM packages if: github.event_name == 'push' - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: ${{ steps.vars.outputs.zip_file }} - path: ${{ steps.vars.outputs.zip_file }} + path: | + build/pkg/rpm/rpmbuild/RPMS/ + build/pkg/rpm/rpmbuild/SRPMS/ From dbcf0fd49e109f076ba0d3414f31526d46a42627 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 8 Feb 2022 15:45:36 +0100 Subject: [PATCH 16/31] CI: improve name of ZIP file with RPM/DEB artifacts --- .github/workflows/packages.yml | 37 ++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 7d70fa95..2f34046b 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -23,12 +23,18 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Define global variables - run: echo "::set-output name=zip_file::ipfixcol2-${IMAGE//:/}-$GITHUB_SHA.zip" - shell: bash - env: - IMAGE: ${{ matrix.image }} - id: vars + - name: Define variables + uses: actions/github-script@v5 + with: + script: | + const sha = context.sha.substring(0, 8); + const image = `${{ matrix.image }}` + const distro = image.split('/').pop().replace(/:/g,'_'); + const zip = `ipfixcol2-${distro}-${sha}`; + core.exportVariable('ZIP_FILE', zip); + - name: Prepare environment + run: | + mkdir -p build/libfds_repo # Dependencies --------------------------------------------------------------------------- - name: Install dependencies for libfds and IPFIXcol2 (Ubuntu/Debian) @@ -71,7 +77,7 @@ jobs: if: github.event_name == 'push' uses: actions/upload-artifact@v2 with: - name: ${{ steps.vars.outputs.zip_file }} + name: ${{ env.ZIP_FILE }} path: | build/pkg/deb/debbuild/*.deb build/pkg/deb/debbuild/*.ddeb @@ -91,13 +97,18 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Prepare environment and variables + - name: Define variables + uses: actions/github-script@v5 + with: + script: | + const sha = context.sha.substring(0, 8); + const image = `${{ matrix.image }}` + const distro = image.split('/').pop().replace(/:/g,'_'); + const zip = `ipfixcol2-${distro}-${sha}`; + core.exportVariable('ZIP_FILE', zip); + - name: Prepare environment run: | - echo "::set-output name=zip_file::ipfixcol2-${IMAGE//:/}-$GITHUB_SHA.zip" mkdir -p build/libfds_repo - env: - IMAGE: ${{ matrix.image }} - id: vars # Dependencies --------------------------------------------------------------------------- - name: Enable additional repositories (CentOS 8) @@ -149,7 +160,7 @@ jobs: if: github.event_name == 'push' uses: actions/upload-artifact@v2 with: - name: ${{ steps.vars.outputs.zip_file }} + name: ${{ env.ZIP_FILE }} path: | build/pkg/rpm/rpmbuild/RPMS/ build/pkg/rpm/rpmbuild/SRPMS/ From 5c608f648d33fb1d8e08c8d0b699128133df04f9 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 8 Feb 2022 16:03:35 +0100 Subject: [PATCH 17/31] CI: replace CentOS 8 with CentOS 8 Stream CentOS 8 is not supported by Red Hat anymore and its repositories are broken. --- .github/workflows/main.yml | 15 +++++++++++---- .github/workflows/packages.yml | 10 ++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 09ca96fe..f6dfb6e3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,14 @@ jobs: strategy: fail-fast: false matrix: - image: ['ubuntu:18.04', 'ubuntu:20.04', 'debian:stretch', 'debian:buster', 'debian:bullseye', 'centos:7', 'centos:8'] + image: + - 'ubuntu:18.04' + - 'ubuntu:20.04' + - 'debian:stretch' + - 'debian:buster' + - 'debian:bullseye' + - 'centos:7' + - 'quay.io/centos/centos:stream8' name: Build on ${{ matrix.image }} container: ${{ matrix.image }} @@ -32,13 +39,13 @@ jobs: apt-get -y install librdkafka-dev env: DEBIAN_FRONTEND: noninteractive - - name: Enable additional repositories (CentOS 8) - if: startsWith(matrix.image, 'centos:8') + - name: Enable additional repositories (CentOS Steam) + if: contains(matrix.image, 'centos:stream') run: | dnf -y install 'dnf-command(config-manager)' dnf config-manager --set-enabled appstream powertools - name: Install dependencies for libfds and IPFIXcol2 (CentOS) - if: startsWith(matrix.image, 'centos') + if: contains(matrix.image, 'centos') run: | yum -y install epel-release yum -y install git gcc gcc-c++ cmake make libxml2-devel lz4-devel libzstd-devel diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 2f34046b..2dc7c0f5 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -90,7 +90,9 @@ jobs: strategy: fail-fast: false matrix: - image: ['centos:7', 'centos:8'] + image: + - 'centos:7' + - 'quay.io/centos/centos:stream8' name: Build RPMs on ${{ matrix.image }} container: ${{ matrix.image }} @@ -111,13 +113,13 @@ jobs: mkdir -p build/libfds_repo # Dependencies --------------------------------------------------------------------------- - - name: Enable additional repositories (CentOS 8) - if: startsWith(matrix.image, 'centos:8') + - name: Enable additional repositories (CentOS Stream) + if: contains(matrix.image, 'centos:stream') run: | dnf -y install 'dnf-command(config-manager)' dnf config-manager --set-enabled appstream powertools - name: Install dependencies for libfds and IPFIXcol2 (CentOS) - if: startsWith(matrix.image, 'centos') + if: contains(matrix.image, 'centos') run: | yum -y install epel-release yum -y install git gcc gcc-c++ cmake make libxml2-devel lz4-devel libzstd-devel From f6f5f83f6fd068b9ab9448f79b1f4ea384a89fe4 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 8 Feb 2022 17:20:40 +0100 Subject: [PATCH 18/31] CI: remove unused Fedora installation steps --- .github/workflows/main.yml | 5 ----- .github/workflows/packages.yml | 5 ----- 2 files changed, 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f6dfb6e3..fae45ed9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -51,11 +51,6 @@ jobs: yum -y install git gcc gcc-c++ cmake make libxml2-devel lz4-devel libzstd-devel yum -y install zlib-devel pkgconfig librdkafka-devel yum -y install python3-docutils || yum -y install python-docutils - - name: Install dependencies for libfds and IPFIXcol2 (Fedora) - if: startsWith(matrix.image, 'fedora') - run: | - dnf -y install git gcc gcc-c++ cmake make libxml2-devel lz4-devel libzstd-devel - dnf -y install python3-docutils zlib-devel pkgconfig librdkafka-devel # Build libfds library ------------------------------------------------------------------ # Note: Master against master branch. Otherwise against debug branch. diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 2dc7c0f5..e9d6697c 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -125,11 +125,6 @@ jobs: yum -y install git gcc gcc-c++ cmake make libxml2-devel lz4-devel libzstd-devel yum -y install zlib-devel pkgconfig rpm-build librdkafka-devel yum -y install python3-docutils || yum -y install python-docutils - - name: Install depedencies for libfds and IPFIXcol2 (Fedora) - if: startsWith(matrix.image, 'fedora') - run: | - dnf -y install git gcc gcc-c++ cmake make libxml2-devel lz4-devel libzstd-devel - dnf -y install python3-docutils zlib-devel pkgconfig rpm-build librdkafka-devel # Build LIBFDS RPM package --------------------------------------------------------------- - name: Checkout libfds library - master branch From a3b7dc706c69acf86da82f8e116ee0d3c7583539 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 8 Feb 2022 17:34:55 +0100 Subject: [PATCH 19/31] CI: introduce support for Oracle Linux 8 --- .github/workflows/main.yml | 19 +++++++++++++++++-- .github/workflows/packages.yml | 19 +++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fae45ed9..d4023749 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,11 +23,11 @@ jobs: - 'debian:bullseye' - 'centos:7' - 'quay.io/centos/centos:stream8' + - 'oraclelinux:8' name: Build on ${{ matrix.image }} container: ${{ matrix.image }} steps: - - uses: actions/checkout@v2 # Dependencies --------------------------------------------------------------------------- - name: Install dependencies for libfds and IPFIXcol2 (Ubuntu/Debian) @@ -44,14 +44,29 @@ jobs: run: | dnf -y install 'dnf-command(config-manager)' dnf config-manager --set-enabled appstream powertools - - name: Install dependencies for libfds and IPFIXcol2 (CentOS) + - name: Enable additional repositories (Oracle Linux) + if: contains(matrix.image, 'oraclelinux') + run: | + dnf -y install 'dnf-command(config-manager)' + dnf config-manager --set-enabled ol8_appstream ol8_codeready_builder + - name: Enable EPEL (CentOS) if: contains(matrix.image, 'centos') run: | yum -y install epel-release + - name: Enable EPEL (OracleLinux) + if: contains(matrix.image, 'oraclelinux') + run: | + dnf -y install oracle-epel-release-el8 + - name: Install dependencies for libfds and IPFIXcol2 (CentOS, Oracle Linux) + if: contains(matrix.image, 'centos') || contains(matrix.image, 'oraclelinux') + run: | yum -y install git gcc gcc-c++ cmake make libxml2-devel lz4-devel libzstd-devel yum -y install zlib-devel pkgconfig librdkafka-devel yum -y install python3-docutils || yum -y install python-docutils + # Checkout repository -------------------------------------------------------------------- + - uses: actions/checkout@v2 + # Build libfds library ------------------------------------------------------------------ # Note: Master against master branch. Otherwise against debug branch. - name: Checkout libfds library - master branch diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index e9d6697c..8fc28285 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -93,12 +93,12 @@ jobs: image: - 'centos:7' - 'quay.io/centos/centos:stream8' + - 'oraclelinux:8' name: Build RPMs on ${{ matrix.image }} container: ${{ matrix.image }} steps: - - uses: actions/checkout@v2 - name: Define variables uses: actions/github-script@v5 with: @@ -118,14 +118,29 @@ jobs: run: | dnf -y install 'dnf-command(config-manager)' dnf config-manager --set-enabled appstream powertools - - name: Install dependencies for libfds and IPFIXcol2 (CentOS) + - name: Enable additional repositories (Oracle Linux) + if: contains(matrix.image, 'oraclelinux') + run: | + dnf -y install 'dnf-command(config-manager)' + dnf config-manager --set-enabled ol8_appstream ol8_codeready_builder + - name: Enable EPEL (CentOS) if: contains(matrix.image, 'centos') run: | yum -y install epel-release + - name: Enable EPEL (OracleLinux) + if: contains(matrix.image, 'oraclelinux') + run: | + dnf -y install oracle-epel-release-el8 + - name: Install dependencies for libfds and IPFIXcol2 (CentOS, Oracle Linux) + if: contains(matrix.image, 'centos') || contains(matrix.image, 'oraclelinux') + run: | yum -y install git gcc gcc-c++ cmake make libxml2-devel lz4-devel libzstd-devel yum -y install zlib-devel pkgconfig rpm-build librdkafka-devel yum -y install python3-docutils || yum -y install python-docutils + # Checkout repository -------------------------------------------------------------------- + - uses: actions/checkout@v2 + # Build LIBFDS RPM package --------------------------------------------------------------- - name: Checkout libfds library - master branch if: github.ref == 'refs/heads/master' From a5028347d09e3ced80ccb84bd47c2a1a583ec99b Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 8 Feb 2022 17:50:42 +0100 Subject: [PATCH 20/31] README: improve installation steps for CentOS and Oracle Linux --- README.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index ef3b3b17..d3df01b1 100644 --- a/README.rst +++ b/README.rst @@ -86,15 +86,23 @@ Second, install build dependencies of the collector yum install gcc gcc-c++ cmake make python3-docutils zlib-devel librdkafka-devel # Optionally: doxygen pkgconfig -* Note: latest systems (e.g. Fedora/CentOS 8) use ``dnf`` instead of ``yum``. +* Note: latest systems (e.g. Fedora/CentOS Stream 8) use ``dnf`` instead of ``yum``. * Note: package ``python3-docutils`` may by also named as ``python-docutils`` or ``python2-docutils`` * Note: package ``pkgconfig`` may by also named as ``pkg-config`` -* Note: CentOS 8 requires additional system repositories (``appstream`` and ``powertools``) to be enabled: +* Note: CentOS Stream 8 usually requires additional system repositories to be enabled: .. code-block:: + dnf -y install epel-release dnf config-manager --set-enabled appstream powertools +* Note: Oracle Linux 8 usually requires additional system repositories to be enabled: + +.. code-block:: + + dnf -y install oracle-epel-release-el8 + dnf config-manager --set-enabled ol8_appstream ol8_codeready_builder + **Debian/Ubuntu:** .. code-block:: From d4535b4ab2a606075e1e0540f368df1bf02e0c0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Hut=C3=A1k?= Date: Tue, 8 Feb 2022 18:11:27 +0100 Subject: [PATCH 21/31] Sedlak forwarder rework (#60) * Forwarder: refactor v1 * Forwarder: improve config * Forwarder: improve main * Forwarder: removed old files * Forwarder: updated CMakeLists * Forwarder: added common * Forwarder: added forwarder * Forwarder: added host * Forwarder: added connection * Forwarder: added sender * Forwarder: added message * Forwarder: improve docs * Forwarder: resend templates when connection reconnects * Forwarder: skip hosts with waiting transfers * Forwarder: changed how reconnections work * Forwarder: indicate lost messages in SendToAll mode * Forwarder: fixes in message creation * Forwarder: use MSG_DONTSIGNAL when sending over socket * Forwarder: remove debugging printfs * Forwarder: minor fixes * Forwarder: atomic sockfd when reconnecting from another thread * Forwarder: atomic finished flag * Forwarder: drop unfinished transfers on connection lost for TCP * Forwarder: fix build on gcc 4.8 * Forwarder: skip data sets with unknown templates * Forwarder: add non-blocking socket connect * Forwarder: set export time to current time when forwarding * Forwarder: minor fixes in Connection * Forwarder: non-blocking socket connect v2 * Forwarder: fix reconnector Co-authored-by: Michal Sedlak --- src/plugins/output/forwarder/CMakeLists.txt | 27 +- src/plugins/output/forwarder/README.rst | 55 +- src/plugins/output/forwarder/TODO.txt | 5 - src/plugins/output/forwarder/src/Config.cpp | 307 +++++++++ src/plugins/output/forwarder/src/Config.h | 122 ++++ .../output/forwarder/src/Connection.cpp | 240 +++++-- src/plugins/output/forwarder/src/Connection.h | 173 ++++-- .../output/forwarder/src/ConnectionBuffer.h | 221 ------- .../forwarder/src/ConnectionManager.cpp | 170 ----- .../output/forwarder/src/ConnectionParams.h | 136 ---- .../output/forwarder/src/Forwarder.cpp | 133 ++++ src/plugins/output/forwarder/src/Forwarder.h | 430 ++----------- src/plugins/output/forwarder/src/Host.cpp | 168 +++++ src/plugins/output/forwarder/src/Host.h | 119 ++++ src/plugins/output/forwarder/src/Message.cpp | 185 ++++++ src/plugins/output/forwarder/src/Message.h | 169 +++++ .../output/forwarder/src/MessageBuilder.h | 150 ----- src/plugins/output/forwarder/src/Sender.cpp | 253 ++++++++ src/plugins/output/forwarder/src/Sender.h | 110 ++++ src/plugins/output/forwarder/src/common.cpp | 98 +++ src/plugins/output/forwarder/src/common.h | 157 +++++ src/plugins/output/forwarder/src/config.h | 306 --------- .../forwarder/src/connector/Connector.cpp | 583 ++++++++++++++++++ .../forwarder/src/connector/Connector.h | 180 ++++++ .../forwarder/src/connector/FutureSocket.cpp | 70 +++ .../FutureSocket.h} | 71 +-- .../{IPFIXMessage.h => connector/Pipe.cpp} | 88 +-- .../src/{SyncPipe.h => connector/Pipe.h} | 69 +-- src/plugins/output/forwarder/src/main.cpp | 70 ++- 29 files changed, 3198 insertions(+), 1667 deletions(-) delete mode 100644 src/plugins/output/forwarder/TODO.txt create mode 100644 src/plugins/output/forwarder/src/Config.cpp create mode 100644 src/plugins/output/forwarder/src/Config.h delete mode 100644 src/plugins/output/forwarder/src/ConnectionBuffer.h delete mode 100644 src/plugins/output/forwarder/src/ConnectionManager.cpp delete mode 100644 src/plugins/output/forwarder/src/ConnectionParams.h create mode 100644 src/plugins/output/forwarder/src/Forwarder.cpp create mode 100644 src/plugins/output/forwarder/src/Host.cpp create mode 100644 src/plugins/output/forwarder/src/Host.h create mode 100644 src/plugins/output/forwarder/src/Message.cpp create mode 100644 src/plugins/output/forwarder/src/Message.h delete mode 100644 src/plugins/output/forwarder/src/MessageBuilder.h create mode 100644 src/plugins/output/forwarder/src/Sender.cpp create mode 100644 src/plugins/output/forwarder/src/Sender.h create mode 100644 src/plugins/output/forwarder/src/common.cpp create mode 100644 src/plugins/output/forwarder/src/common.h delete mode 100644 src/plugins/output/forwarder/src/config.h create mode 100644 src/plugins/output/forwarder/src/connector/Connector.cpp create mode 100644 src/plugins/output/forwarder/src/connector/Connector.h create mode 100644 src/plugins/output/forwarder/src/connector/FutureSocket.cpp rename src/plugins/output/forwarder/src/{ConnectionManager.h => connector/FutureSocket.h} (61%) rename src/plugins/output/forwarder/src/{IPFIXMessage.h => connector/Pipe.cpp} (64%) rename src/plugins/output/forwarder/src/{SyncPipe.h => connector/Pipe.h} (68%) diff --git a/src/plugins/output/forwarder/CMakeLists.txt b/src/plugins/output/forwarder/CMakeLists.txt index 2bec16c3..66ee28b6 100644 --- a/src/plugins/output/forwarder/CMakeLists.txt +++ b/src/plugins/output/forwarder/CMakeLists.txt @@ -1,19 +1,30 @@ # Create a linkable module add_library(forwarder-output MODULE src/main.cpp - src/config.h + src/Config.h + src/Config.cpp src/Forwarder.h - src/ConnectionManager.h - src/ConnectionManager.cpp - src/ConnectionParams.h + src/Forwarder.cpp src/Connection.h src/Connection.cpp - src/ConnectionBuffer.h - src/SyncPipe.h - src/IPFIXMessage.h - src/MessageBuilder.h + src/Host.cpp + src/Host.h + src/common.h + src/common.cpp + src/Message.h + src/Message.cpp + src/Sender.h + src/Sender.cpp + src/connector/Connector.h + src/connector/Connector.cpp + src/connector/FutureSocket.h + src/connector/FutureSocket.cpp + src/connector/Pipe.h + src/connector/Pipe.cpp ) +target_include_directories(forwarder-output PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src) + install( TARGETS forwarder-output LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" diff --git a/src/plugins/output/forwarder/README.rst b/src/plugins/output/forwarder/README.rst index 93fdde4c..aaeef7a6 100644 --- a/src/plugins/output/forwarder/README.rst +++ b/src/plugins/output/forwarder/README.rst @@ -1,7 +1,7 @@ Forwarder (output plugin) [Preview] =================================== -This plugin allows forwarding incoming IPFIX messages to other collector in various modes. +This plugin allows forwarding incoming flow records in IPFIX form to other collector in various modes. It can be used to broadcast messages to multiple collectors (e.g. a main and a backup collector), or to distribute messages across multiple collectors (e.g. for load balancing). @@ -20,13 +20,18 @@ Example configuration Subcollector 1 -
127.0.0.1
- 4751 +
192.168.1.2
+ 4739 Subcollector 2 +
192.168.1.3
+ 4739 + + + Subcollector 3
localhost
- 4752 + 4739 @@ -36,42 +41,48 @@ Parameters ---------- :``mode``: - The forwarding mode; round robin (messages are sent to one host at time and hosts are cycled through) or all (messages are broadcasted to all hosts) + Flow distribution mode. **RoundRobin** (each record will be delivered to one of hosts) or **All** (each record will be delivered to all hosts). [values: RoundRobin/All] :``protocol``: - The transport protocol to use + The transport protocol to use. [values: TCP/UDP] -:``connectionBufferSize``: - Size of the buffer of each connection (Warning: number of connections = number of input exporters * number of hosts) - [value: number of bytes, default: 4194304] +:``templatesResendSecs``: + Send templates again every N seconds (UDP only). + [value: number of seconds, default: 600, 0 = never] -:``templateRefreshIntervalSecs``: - Send templates again every N seconds (UDP only) - [value: number of seconds, default: 600] +:``templatesResendPkts``: + Send templates again every N packets (UDP only). + [value: number of packets, default: 5000, 0 = never] -:``templateRefreshIntervalBytes``: - Send templates again every N bytes (UDP only) - [value: number of bytes, default: 5000000] +:``reconnectSecs``: + Attempt to reconnect every N seconds in case the connection drops (TCP only). + [value: number of seconds, default: 10, 0 = don't wait] -:``reconnectIntervalSecs``: - Attempt to reconnect every N seconds in case the connection drops (TCP only) - [value: number of seconds, default: 10] +:``premadeConnections``: + Keep N connections open with each host so there is no delay in connecting once a connection is needed. + [value: number of connections, default: 5] :``hosts``: - The receiving hosts + The receiving hosts. :``host``: :``name``: - Optional identification of the host + Optional identification of the host. [value: string, default:
:] :``address``: - The address of the host + The address of the host. [value: IPv4/IPv6 address or a hostname] :``port``: - The port to connect to + The port to connect to. [value: port number] +Known limitations +----------------- + +Export time of IPFIX messages is set to the current time when forwarding. This may cause issues with data fields using deltaTime relative to the export time! + + diff --git a/src/plugins/output/forwarder/TODO.txt b/src/plugins/output/forwarder/TODO.txt deleted file mode 100644 index 37631f98..00000000 --- a/src/plugins/output/forwarder/TODO.txt +++ /dev/null @@ -1,5 +0,0 @@ -* Template withdrawals -* More effective way of handling template changes - currently all the templates are being sent again every time any change in templates is detected -* Message MTU -* Possible bug: when testing, a small number of data records seems to be lost (something like 20 out of 1,000,000) -* Connection buffer size \ No newline at end of file diff --git a/src/plugins/output/forwarder/src/Config.cpp b/src/plugins/output/forwarder/src/Config.cpp new file mode 100644 index 00000000..1c2b93b6 --- /dev/null +++ b/src/plugins/output/forwarder/src/Config.cpp @@ -0,0 +1,307 @@ +/** + * \file src/plugins/output/forwarder/src/Config.cpp + * \author Michal Sedlak + * \brief Plugin configuration implementation + * \date 2021 + */ + +/* Copyright (C) 2021 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include "Config.h" + +#include +#include +#include +#include +#include + +/// +/// Config schema definition +/// + +enum { + MODE, + PROTOCOL, + RECONNECT_SECS, + TEMPLATES_RESEND_SECS, + TEMPLATES_RESEND_PKTS, + CONNECTION_BUFFER_SIZE, + HOSTS, + HOST, + NAME, + ADDRESS, + PORT, + PREMADE_CONNECTIONS +}; + +static fds_xml_args host_schema[] = { + FDS_OPTS_ELEM(NAME , "name" , FDS_OPTS_T_STRING, FDS_OPTS_P_OPT), + FDS_OPTS_ELEM(ADDRESS, "address", FDS_OPTS_T_STRING, 0 ), + FDS_OPTS_ELEM(PORT , "port" , FDS_OPTS_T_UINT , 0 ), + FDS_OPTS_END +}; + +static fds_xml_args hosts_schema[] = { + FDS_OPTS_NESTED(HOST, "host", host_schema, FDS_OPTS_P_MULTI), + FDS_OPTS_END +}; + +static fds_xml_args params_schema[] = { + FDS_OPTS_ROOT ("params"), + FDS_OPTS_ELEM (MODE , "mode" , FDS_OPTS_T_STRING, 0 ), + FDS_OPTS_ELEM (PROTOCOL , "protocol" , FDS_OPTS_T_STRING, 0 ), + FDS_OPTS_ELEM (TEMPLATES_RESEND_SECS, "templatesResendSecs", FDS_OPTS_T_UINT , FDS_OPTS_P_OPT), + FDS_OPTS_ELEM (TEMPLATES_RESEND_PKTS, "templatesResendPkts", FDS_OPTS_T_UINT , FDS_OPTS_P_OPT), + FDS_OPTS_ELEM (RECONNECT_SECS , "reconnectSecs" , FDS_OPTS_T_UINT , FDS_OPTS_P_OPT), + FDS_OPTS_ELEM (PREMADE_CONNECTIONS , "premadeConnections" , FDS_OPTS_T_UINT , FDS_OPTS_P_OPT), + FDS_OPTS_NESTED(HOSTS , "hosts" , hosts_schema , 0 ), + FDS_OPTS_END +}; + +/// +/// Config definition +/// + +/** + * \brief Create a new configuration + * \param[in] params XML configuration of JSON plugin + * \throw runtime_error in case of invalid configuration + */ +Config::Config(const char *xml_config) +{ + set_defaults(); + + auto parser = std::unique_ptr(fds_xml_create(), &fds_xml_destroy); + if (!parser) { + throw std::runtime_error("Failed to create an XML parser!"); + } + + if (fds_xml_set_args(parser.get(), params_schema) != FDS_OK) { + throw std::runtime_error("Failed to parse the description of an XML document!"); + } + + fds_xml_ctx_t *params_ctx = fds_xml_parse_mem(parser.get(), xml_config, true); + if (!params_ctx) { + std::string err = fds_xml_last_err(parser.get()); + throw std::runtime_error("Failed to parse the configuration: " + err); + } + + try { + parse_params(params_ctx); + ensure_valid(); + } catch (const std::invalid_argument &ex) { + throw std::runtime_error("Config params error: " + std::string(ex.what())); + } +} + +void +Config::parse_params(fds_xml_ctx_t *params_ctx) +{ + const fds_xml_cont *content; + + while (fds_xml_next(params_ctx, &content) != FDS_EOC) { + + switch (content->id) { + + case MODE: + if (strcasecmp(content->ptr_string, "roundrobin") == 0) { + this->forward_mode = ForwardMode::ROUNDROBIN; + + } else if (strcasecmp(content->ptr_string, "all") == 0) { + this->forward_mode = ForwardMode::SENDTOALL; + + } else { + throw std::invalid_argument("mode must be one of: 'RoundRobin', 'All'"); + + } + break; + + case PROTOCOL: + if (strcasecmp(content->ptr_string, "tcp") == 0) { + this->protocol = Protocol::TCP; + + } else if (strcasecmp(content->ptr_string, "udp") == 0) { + this->protocol = Protocol::UDP; + + } else { + throw std::invalid_argument("protocol must be one of: 'TCP', 'UDP'"); + + } + break; + + case HOSTS: + parse_hosts(content->ptr_ctx); + break; + + case TEMPLATES_RESEND_SECS: + this->tmplts_resend_secs = content->val_uint; + break; + + case TEMPLATES_RESEND_PKTS: + this->tmplts_resend_pkts = content->val_uint; + break; + + case RECONNECT_SECS: + this->reconnect_secs = content->val_uint; + break; + + case PREMADE_CONNECTIONS: + this->nb_premade_connections = content->val_uint; + break; + + default: assert(0); + } + } +} + +void +Config::parse_hosts(fds_xml_ctx_t *hosts_ctx) +{ + const fds_xml_cont *content; + + while (fds_xml_next(hosts_ctx, &content) != FDS_EOC) { + assert(content->id == HOST); + parse_host(content->ptr_ctx); + } +} + +void +Config::parse_host(fds_xml_ctx_t *host_ctx) +{ + HostConfig host; + + const fds_xml_cont *content; + + while (fds_xml_next(host_ctx, &content) != FDS_EOC) { + + switch (content->id) { + + case NAME: + host.name = std::string(content->ptr_string); + break; + + case ADDRESS: + host.address = std::string(content->ptr_string); + break; + + case PORT: + if (content->val_uint > UINT16_MAX) { + throw std::invalid_argument("invalid host port " + std::to_string(content->val_uint)); + } + + host.port = static_cast(content->val_uint); + break; + } + } + + if (host.name.empty()) { + host.name = host.address + ":" + std::to_string(host.port); + } + + if (host_exists(host)) { + throw std::invalid_argument("duplicate host " + host.address + ":" + std::to_string(host.port)); + } + + hosts.push_back(host); +} + +void +Config::set_defaults() +{ + this->tmplts_resend_secs = 10 * 60; + this->tmplts_resend_pkts = 5000; + this->reconnect_secs = 10; + this->nb_premade_connections = 5; +} + +void +Config::ensure_valid() +{ + for (auto &host : hosts) { + + if (!can_resolve_host(host)) { + throw std::invalid_argument("cannot resolve host address " + host.address); + } + + } +} + +bool +Config::host_exists(HostConfig host) +{ + for (auto &host_ : hosts) { + + if (host.address == host_.address && host.port == host_.port) { + return true; + } + + } + + return false; +} + +bool +Config::can_resolve_host(HostConfig host) +{ + addrinfo *info; + + addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + + switch (protocol) { + case Protocol::TCP: + hints.ai_protocol = IPPROTO_TCP; + hints.ai_socktype = SOCK_STREAM; + break; + + case Protocol::UDP: + hints.ai_protocol = IPPROTO_UDP; + hints.ai_socktype = SOCK_DGRAM; + break; + + default: assert(0); + } + + int ret = getaddrinfo(host.address.c_str(), std::to_string(host.port).c_str(), &hints, &info); + + if (ret == 0) { + freeaddrinfo(info); + return true; + + } else { + return false; + } +} \ No newline at end of file diff --git a/src/plugins/output/forwarder/src/Config.h b/src/plugins/output/forwarder/src/Config.h new file mode 100644 index 00000000..9eda1651 --- /dev/null +++ b/src/plugins/output/forwarder/src/Config.h @@ -0,0 +1,122 @@ +/** + * \file src/plugins/output/forwarder/src/Config.h + * \author Michal Sedlak + * \brief Plugin configuration header + * \date 2021 + */ + +/* Copyright (C) 2021 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ +#pragma once + +#include + +#include +#include + +#include "common.h" + +/// The forwarding mode +enum class ForwardMode { + UNASSIGNED, + SENDTOALL, /// Every message is forwarded to all of the hosts + ROUNDROBIN /// Only one host receives each message, next host is selected every message +}; + +struct HostConfig { + /// The displayed name of the host, purely informational + std::string name; + /// The address of the host, IP address or a hostname + std::string address; + /// The port of the host + uint16_t port; +}; + +/// The config to be passed to the forwarder +class Config +{ +public: + /// The transport protocol to be used for connection to the hosts + Protocol protocol; + /// The mode of forwarding messages + ForwardMode forward_mode; + /// Connection parameters of the hosts the data will be forwarded to + std::vector hosts; + /// The number of packets sent between sending refresh of templates (whichever happens first, packets or secs) + unsigned int tmplts_resend_pkts; + /// The number of seconds elapsed between sending refresh of templates (whichever happens first, packets or secs) + unsigned int tmplts_resend_secs; + /// The number of seconds to wait before trying to reconnect when using a TCP connection + unsigned int reconnect_secs; + /// Number of premade connections to keep + unsigned int nb_premade_connections; + + Config() {}; + + /** + * \brief Create a new configuration + * \param[in] params XML configuration of the plugin + * \throw runtime_error in case of invalid configuration + */ + Config(const char *xml_config); + +private: + /// Parse the element + void + parse_params(fds_xml_ctx_t *params_ctx); + + /// Parse the element + void + parse_hosts(fds_xml_ctx_t *hosts_ctx); + + /// Parse the element + void + parse_host(fds_xml_ctx_t *host_ctx); + + /// Set default config values + void + set_defaults(); + + /// Ensure the configuration is valid or throw an exception + void + ensure_valid(); + + /// Check if a host already exists in the vector of hosts + bool + host_exists(HostConfig host); + + /// Check if the host address can be resolved + bool + can_resolve_host(HostConfig host); +}; \ No newline at end of file diff --git a/src/plugins/output/forwarder/src/Connection.cpp b/src/plugins/output/forwarder/src/Connection.cpp index b7d2c5b6..0bdb55f3 100644 --- a/src/plugins/output/forwarder/src/Connection.cpp +++ b/src/plugins/output/forwarder/src/Connection.cpp @@ -1,7 +1,7 @@ /** * \file src/plugins/output/forwarder/src/Connection.cpp * \author Michal Sedlak - * \brief Buffered socket connection + * \brief Connection class implementation * \date 2021 */ @@ -41,90 +41,222 @@ #include "Connection.h" +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + #include -Connection::Connection(ConnectionManager &manager, ConnectionParams params, long buffer_size) -: manager(manager) -, params(params) -, buffer(buffer_size) +#include "Message.h" + +Connection::Connection(const std::string &ident, ConnectionParams con_params, ipx_ctx_t *log_ctx, + unsigned int tmplts_resend_pkts, unsigned int tmplts_resend_secs, + Connector &connector) : + m_ident(ident), + m_con_params(con_params), + m_log_ctx(log_ctx), + m_tmplts_resend_pkts(tmplts_resend_pkts), + m_tmplts_resend_secs(tmplts_resend_secs), + m_connector(connector) { } -bool +void Connection::connect() { - if (sockfd >= 0) { - ::close(sockfd); + assert(m_sockfd.get() < 0); + m_future_socket = m_connector.get(m_con_params); +} + +void +Connection::forward_message(ipx_msg_ipfix_t *msg) +{ + assert(check_connected()); + + Sender &sender = get_or_create_sender(msg); + try { + sender.process_message(msg); + + } catch (const ConnectionError &err) { + // In case connection was lost, we have to resend templates when it reconnects + sender.clear_templates(); + throw err; } - sockfd = params.make_socket(); - return sockfd >= 0; } -std::unique_lock -Connection::begin_write() +void +Connection::lose_message(ipx_msg_ipfix_t *msg) { - return std::unique_lock(buffer_mutex); + Sender &sender = get_or_create_sender(msg); + sender.lose_message(msg); } -bool -Connection::write(void *data, long length) +void +Connection::advance_transfers() { - return buffer.write((uint8_t *)data, length); + assert(check_connected()); + + IPX_CTX_DEBUG(m_log_ctx, "Waiting transfers on connection %s: %zu", m_ident.c_str(), m_transfers.size()); + + for (auto it = m_transfers.begin(); it != m_transfers.end(); ) { + + Transfer &transfer = *it; + + assert(transfer.data.size() <= UINT16_MAX); // The transfer consists of one IPFIX message which cannot be larger + + ssize_t ret = send(m_sockfd.get(), &transfer.data[transfer.offset], + transfer.data.size() - transfer.offset, MSG_DONTWAIT | MSG_NOSIGNAL); + + check_socket_error(ret); + + size_t sent = std::max(0, ret); + IPX_CTX_DEBUG(m_log_ctx, "Sent %zu/%zu B to %s", sent, transfer.data.size(), m_ident.c_str()); + + // Is the transfer done? + if (transfer.offset + sent == transfer.data.size()) { + it = m_transfers.erase(it); + // Remove the transfer and continue with the next one + + } else { + transfer.offset += sent; + + // Finish, cannot advance next transfer before the one before it is fully sent + break; + } + } } -void -Connection::rollback_write() +bool +Connection::check_connected() { - buffer.rollback(); + if (m_sockfd.get() >= 0) { + return true; + } + + if (m_future_socket && m_future_socket->ready()) { + m_sockfd = m_future_socket->retrieve(); + m_future_socket = nullptr; + return true; + } + + return false; } -long -Connection::writeable() + +static Transfer +make_transfer(const std::vector &parts, uint16_t offset, uint16_t total_length) { - return buffer.writeable(); + uint16_t length = total_length - offset; + + // Find first unfinished part + size_t i = 0; + + while (offset >= parts[i].iov_len) { + offset -= parts[i].iov_len; + i++; + } + + // Copy the unfinished portion + std::vector buffer(length); //NOTE: We might want to do this more effectively... + uint16_t buffer_pos = 0; + + for (; i < parts.size(); i++) { + memcpy(buffer.data() + buffer_pos, &((uint8_t *) parts[i].iov_base)[offset], parts[i].iov_len - offset); + buffer_pos += parts[i].iov_len - offset; + offset = 0; + } + + // Create the transfer + Transfer transfer; + transfer.data = std::move(buffer); + transfer.offset = 0; + + return transfer; } -void -Connection::commit_write() +void +Connection::store_unfinished_transfer(Message &msg, uint16_t offset) { - buffer.commit(); - manager.pipe.notify(); - has_data_to_send = buffer.readable(); + Transfer transfer = make_transfer(msg.parts(), offset, msg.length()); + + IPX_CTX_DEBUG(m_log_ctx, "Storing unfinished transfer of %" PRIu16 " bytes in connection to %s", + msg.length() - offset, m_ident.c_str()); + + m_transfers.push_back(std::move(transfer)); } -bool -Connection::send_some() + +void +Connection::send_message(Message &msg) { - if (params.protocol == TransProto::Udp) { - while (1) { - fds_ipfix_msg_hdr ipfix_header; - if (!buffer.peek(ipfix_header)) { - return true; - } - auto message_length = ntohs(ipfix_header.length); - int ret = buffer.send_data(sockfd, message_length); - if (ret == 0 || !buffer.readable()) { - return true; - } else if (ret < 0) { - return false; - } - } - return true; - } else { - return buffer.send_data(sockfd) >= 0; + // All waiting transfers have to be sent first + if (!m_transfers.empty()) { + store_unfinished_transfer(msg, 0); + return; + } + + std::vector &parts = msg.parts(); + + msghdr hdr; + memset(&hdr, 0, sizeof(hdr)); + hdr.msg_iov = parts.data(); + hdr.msg_iovlen = parts.size(); + + ssize_t ret = sendmsg(m_sockfd.get(), &hdr, MSG_DONTWAIT | MSG_NOSIGNAL); + + check_socket_error(ret); + + size_t sent = std::max(0, ret); + + IPX_CTX_DEBUG(m_log_ctx, "Sent %zu/%" PRIu16 " B to %s", sent, msg.length(), m_ident.c_str()); + + if (sent < msg.length()) { + store_unfinished_transfer(msg, sent); } } -void -Connection::close() +Sender & +Connection::get_or_create_sender(ipx_msg_ipfix_t *msg) { - close_flag = true; - manager.pipe.notify(); + uint32_t odid = ipx_msg_ipfix_get_ctx(msg)->odid; + + if (m_senders.find(odid) == m_senders.end()) { + m_senders.emplace(odid, + std::unique_ptr(new Sender( + [&](Message &msg) { + send_message(msg); + }, + m_con_params.protocol == Protocol::TCP, + m_tmplts_resend_pkts, + m_tmplts_resend_secs))); + } + + Sender &sender = *m_senders[odid].get(); + + return sender; } -Connection::~Connection() +void +Connection::check_socket_error(ssize_t sock_ret) { - if (sockfd >= 0) { - ::close(sockfd); + if (sock_ret < 0 && errno != EWOULDBLOCK && errno != EAGAIN) { + char *errbuf; + ipx_strerror(errno, errbuf); + + IPX_CTX_ERROR(m_log_ctx, "A connection to %s lost! (%s)", m_ident.c_str(), errbuf); + m_sockfd.reset(); + + // All state from the previous connection is lost once new one is estabilished + m_transfers.clear(); + + throw ConnectionError(errbuf); } -} \ No newline at end of file +} diff --git a/src/plugins/output/forwarder/src/Connection.h b/src/plugins/output/forwarder/src/Connection.h index 09af5a29..02c76f1d 100644 --- a/src/plugins/output/forwarder/src/Connection.h +++ b/src/plugins/output/forwarder/src/Connection.h @@ -1,7 +1,7 @@ /** * \file src/plugins/output/forwarder/src/Connection.h * \author Michal Sedlak - * \brief Buffered socket connection + * \brief Connection class header * \date 2021 */ @@ -41,77 +41,144 @@ #pragma once -#include "ConnectionManager.h" -#include "ConnectionParams.h" -#include "ConnectionBuffer.h" - -#include -#include -#include -#include - +#include +#include +#include #include -#include -#include -class ConnectionManager; +#include + +#include "common.h" +#include "connector/Connector.h" +#include "Sender.h" -class Connection -{ -friend class ConnectionManager; +class Connection; +/// An error to be thrown on connection errors +class ConnectionError { public: - /// Flag indicating that the connection was lost and the forwarder needs to resend templates etc. - /// The flag won't be reset when the connection is reestablished! - std::atomic connection_lost_flag { false }; + ConnectionError(std::string message) + : m_message(message) {} + + ConnectionError(std::string message, std::shared_ptr &connection) + : m_message(message), m_connection(&connection) {} + + ConnectionError with_connection(std::shared_ptr &connection) const + { return ConnectionError(m_message, connection); } + + const char *what() const { return m_message.c_str(); } - Connection(ConnectionManager &manager, ConnectionParams params, long buffer_size); - - bool + std::shared_ptr *connection() const { return m_connection; } + +private: + std::string m_message; + std::shared_ptr *m_connection; +}; + +/// A transfer to be sent through the connection +struct Transfer { + /// The data to send + std::vector data; + /// The offset to send from, i.e. the amount of data that was already sent + uint16_t offset; +}; + +/// A class representing one of the connections to the subcollector +/// Each host opens one connection per session +class Connection { +public: + /** + * \brief The constructor + * \param ident The host identification + * \param con_params The connection parameters + * \param log_ctx The logging context + * \param tmplts_resend_pkts Interval in packets after which templates are resend (UDP only) + * \param tmplts_resend_secs Interval in seconds after which templates are resend (UDP only) + */ + Connection(const std::string &ident, ConnectionParams con_params, ipx_ctx_t *log_ctx, + unsigned int tmplts_resend_pkts, unsigned int tmplts_resend_secs, + Connector &connector); + + /// Do not permit copying or moving as the connection holds a raw socket that is closed in the destructor + /// (we could instead implement proper moving and copying behavior, but we don't really need it at the moment) + Connection(const Connection &) = delete; + Connection(Connection &&) = delete; + + /** + * \brief Connect the connection socket + * \throw ConnectionError if the socket couldn't be connected + */ + void connect(); - std::unique_lock - begin_write(); + /** + * \brief Forward an IPFIX message + * \param msg The IPFIX message + */ + void + forward_message(ipx_msg_ipfix_t *msg); + + /** + * \brief Lose an IPFIX message, i.e. update the internal state as if it has been forwarded + * even though it is not being sent + * \param msg The IPFIX message + */ + void + lose_message(ipx_msg_ipfix_t *msg); + + /** + * \brief Advance the unfinished transfers + */ + void + advance_transfers(); + + /** + * \brief Check if the connection socket is currently connected + * \return true or false + */ + bool check_connected(); + + /** + * \brief Get number of transfers still waiting to be transmitted + * \return The number of waiting transfers + */ + size_t waiting_transfers_cnt() const { return m_transfers.size(); } + + /** + * \brief The identification of the connection + */ + const std::string &ident() const { return m_ident; } - bool - write(void *data, long length); +private: + const std::string &m_ident; - bool - send_some(); + ConnectionParams m_con_params; - void - commit_write(); + ipx_ctx_t *m_log_ctx; - void - rollback_write(); + unsigned int m_tmplts_resend_pkts; - long - writeable(); + unsigned int m_tmplts_resend_secs; - void - close(); + UniqueFd m_sockfd; - ~Connection(); + std::shared_ptr m_future_socket; -private: - /// The manager managing this connection - ConnectionManager &manager; + std::unordered_map> m_senders; + + std::vector m_transfers; - /// The parameters to estabilish the connection - ConnectionParams params; + Connector &m_connector; - /// The connection socket - int sockfd = -1; + void + store_unfinished_transfer(Message &msg, uint16_t offset); - /// Buffer for the data to send and a mutex guarding it - /// (buffer will be accessed from sender thread and writer thread) - std::mutex buffer_mutex; - ConnectionBuffer buffer; + void + send_message(Message &msg); - /// Flag indicating whether the buffer has any data to send so we don't have to lock the mutex every time - /// (doesn't need to be atomic because we only set it while holding the mutex) - bool has_data_to_send = false; + Sender & + get_or_create_sender(ipx_msg_ipfix_t *msg); - /// Flag indicating that the connection has been closed and can be disposed of after the data is sent - std::atomic close_flag { false }; + void + check_socket_error(ssize_t sock_ret); }; \ No newline at end of file diff --git a/src/plugins/output/forwarder/src/ConnectionBuffer.h b/src/plugins/output/forwarder/src/ConnectionBuffer.h deleted file mode 100644 index 9db928a0..00000000 --- a/src/plugins/output/forwarder/src/ConnectionBuffer.h +++ /dev/null @@ -1,221 +0,0 @@ -/** - * \file src/plugins/output/forwarder/src/ConnectionBuffer.h - * \author Michal Sedlak - * \brief Ring buffer used by connections - * \date 2021 - */ - -/* Copyright (C) 2021 CESNET, z.s.p.o. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * 3. Neither the name of the Company nor the names of its contributors - * may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * ALTERNATIVELY, provided that this notice is retained in full, this - * product may be distributed under the terms of the GNU General Public - * License (GPL) version 2 or later, in which case the provisions - * of the GPL apply INSTEAD OF those given above. - * - * This software is provided ``as is'', and any express or implied - * warranties, including, but not limited to, the implied warranties of - * merchantability and fitness for a particular purpose are disclaimed. - * In no event shall the company or contributors be liable for any - * direct, indirect, incidental, special, exemplary, or consequential - * damages (including, but not limited to, procurement of substitute - * goods or services; loss of use, data, or profits; or business - * interruption) however caused and on any theory of liability, whether - * in contract, strict liability, or tort (including negligence or - * otherwise) arising in any way out of the use of this software, even - * if advised of the possibility of such damage. - * - */ - -#pragma once - -#include - -#include -#include -#include -#include - -class ConnectionBuffer -{ -public: - ConnectionBuffer(long capacity) - : capacity(capacity) - , buffer(capacity) - {} - - void - rollback() - { - write_offset = read_end_offset; - } - - void - commit() - { - read_end_offset = write_offset; - } - - long - writeable() - { - return writeable_from(write_offset); - } - - bool - write(uint8_t *data, long length) - { - long pos = raw_write_at(write_offset, data, length); - if (pos == -1) { - return false; - } - write_offset = pos; - return true; - } - - template - bool - write(T data) - { - return write((uint8_t *)&data, sizeof(T)); - } - - long - readable() - { - return read_offset > read_end_offset - ? capacity - read_offset + read_end_offset - : read_end_offset - read_offset; - } - - bool - peek(uint8_t *data, long length) - { - if (readable() < length) { - return false; - } - raw_read_at(read_offset, data, length); - return true; - } - - template - bool - peek(T &item) - { - return peek((uint8_t *)&item, sizeof(item)); - } - - int - send_data(int sockfd, long length = -1) - { - if (length == -1) { - length = readable(); - } - iovec iov[2] = {}; - iov[0].iov_len = std::min(cont_readable_from(read_offset), length); - iov[0].iov_base = &buffer[read_offset]; - iov[1].iov_len = length - iov[0].iov_len; - iov[1].iov_base = &buffer[0]; - msghdr msg_hdr = {}; - msg_hdr.msg_iov = iov; - msg_hdr.msg_iovlen = 2; - int ret = sendmsg(sockfd, &msg_hdr, MSG_DONTWAIT | MSG_NOSIGNAL); - if (ret < 0) { - return (errno == EWOULDBLOCK || errno == EAGAIN) ? 0 : ret; - } - read_offset = advance(read_offset, ret); - return ret; - } - -private: - long capacity; - long read_offset = 0; - long read_end_offset = 0; - long write_offset = 0; - std::vector buffer; - - long - advance(long pos, long n) - { - return (pos + n) % capacity; - } - - long - readable_from(long pos) - { - return pos > read_end_offset - ? capacity - pos + read_end_offset - : read_end_offset - pos; - } - - long - cont_readable_from(long pos) - { - return pos > read_end_offset - ? capacity - pos - : read_end_offset - pos; - } - - long - raw_read_at(long pos, uint8_t *data, long length) - { - if (readable_from(pos) < length) { - return -1; - } - long read1 = std::min(cont_readable_from(pos), length); - long read2 = length - read1; - memcpy(&data[0], &buffer[pos], read1); - memcpy(&data[read1], &buffer[advance(pos, read1)], read2); - return advance(pos, length); - } - - long - cont_writeable_from(long pos) - { - return read_offset > pos - ? read_offset - pos - 1 - : (read_offset == 0 ? capacity - pos - 1 : capacity - pos); - } - - long - writeable_from(long pos) - { - return read_offset > pos - ? read_offset - pos - 1 - : capacity - pos + read_offset - 1; - } - - long - raw_write_at(long pos, uint8_t *data, long length) - { - /// WARNING: Does not advance the write offset - if (writeable_from(pos) < length) { - return -1; - } - long write1 = std::min(length, cont_writeable_from(pos)); - long write2 = length - write1; - memcpy(&buffer[pos], &data[0], write1); - memcpy(&buffer[advance(pos, write1)], &data[write1], write2); - return advance(pos, length); - } - - template - bool - raw_write_at(long pos, T data) - { - return raw_write_at(pos, (uint8_t *)&data, sizeof(T)); - } - -}; \ No newline at end of file diff --git a/src/plugins/output/forwarder/src/ConnectionManager.cpp b/src/plugins/output/forwarder/src/ConnectionManager.cpp deleted file mode 100644 index f55dafaf..00000000 --- a/src/plugins/output/forwarder/src/ConnectionManager.cpp +++ /dev/null @@ -1,170 +0,0 @@ -/** - * \file src/plugins/output/forwarder/src/ConnectionManager.cpp - * \author Michal Sedlak - * \brief Connection manager - * \date 2021 - */ - -/* Copyright (C) 2021 CESNET, z.s.p.o. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * 3. Neither the name of the Company nor the names of its contributors - * may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * ALTERNATIVELY, provided that this notice is retained in full, this - * product may be distributed under the terms of the GNU General Public - * License (GPL) version 2 or later, in which case the provisions - * of the GPL apply INSTEAD OF those given above. - * - * This software is provided ``as is'', and any express or implied - * warranties, including, but not limited to, the implied warranties of - * merchantability and fitness for a particular purpose are disclaimed. - * In no event shall the company or contributors be liable for any - * direct, indirect, incidental, special, exemplary, or consequential - * damages (including, but not limited to, procurement of substitute - * goods or services; loss of use, data, or profits; or business - * interruption) however caused and on any theory of liability, whether - * in contract, strict liability, or tort (including negligence or - * otherwise) arising in any way out of the use of this software, even - * if advised of the possibility of such damage. - * - */ - -#include "ConnectionManager.h" - -Connection & -ConnectionManager::add_client(ConnectionParams params) -{ - auto connection_ptr = std::unique_ptr(new Connection(*this, params, connection_buffer_size)); - auto &connection = *connection_ptr; - std::lock_guard guard(mutex); - if (connection.connect()) { - active_connections.push_back(std::move(connection_ptr)); - } else { - reconnect_connections.push_back(std::move(connection_ptr)); - } - return connection; -} - -void -ConnectionManager::send_loop() -{ - int max_fd; - - fd_set pipe_fds; - FD_ZERO(&pipe_fds); - FD_SET(pipe.get_readfd(), &pipe_fds); - - fd_set socket_fds; - FD_ZERO(&socket_fds); - - auto watch_sock = [&](int fd) { - FD_SET(fd, &pipe_fds); - max_fd = std::max(max_fd, fd); - }; - - while (!exit_flag) { - max_fd = pipe.get_readfd(); - FD_ZERO(&socket_fds); - - { - std::lock_guard guard(mutex); - pipe.clear(); - auto it = active_connections.begin(); - while (it != active_connections.end()) { - auto &connection = **it; - if (connection.has_data_to_send) { - std::lock_guard guard(connection.buffer_mutex); - if (connection.send_some()) { - if (connection.buffer.readable()) { - watch_sock(connection.sockfd); - } - connection.has_data_to_send = connection.buffer.readable(); - } else { - connection.connection_lost_flag = true; - reconnect_connections.push_back(std::move(*it)); - it = active_connections.erase(it); - reconnect_cv.notify_one(); - continue; - } - } else { - if (connection.close_flag) { - it = active_connections.erase(it); - continue; - } - } - it++; - } - } - - select(max_fd + 1, &pipe_fds, &socket_fds, NULL, NULL); - } -} - -void -ConnectionManager::reconnect_loop() -{ - while (!exit_flag) { - auto lock = std::unique_lock(mutex); - auto it = reconnect_connections.begin(); - while (it != reconnect_connections.end()) { - auto &connection = **it; - if (connection.connect()) { - active_connections.push_back(std::move(*it)); - it = reconnect_connections.erase(it); - pipe.notify(); - } else { - if (connection.close_flag) { - it = reconnect_connections.erase(it); - continue; - } - it++; - } - } - - if (reconnect_connections.empty()) { - reconnect_cv.wait(lock); - } else { - reconnect_cv.wait_for(lock, std::chrono::seconds(reconnect_interval_secs)); - } - } -} - -void -ConnectionManager::start() -{ - send_thread = std::thread([this]() { send_loop(); }); - reconnect_thread = std::thread([this]() { reconnect_loop(); }); -} - -void -ConnectionManager::stop() -{ - exit_flag = true; - pipe.notify(); - reconnect_cv.notify_one(); - send_thread.join(); - reconnect_thread.join(); -} - -void -ConnectionManager::set_reconnect_interval(int number_of_seconds) -{ - reconnect_interval_secs = number_of_seconds; -} - -void -ConnectionManager::set_connection_buffer_size(long number_of_bytes) -{ - connection_buffer_size = number_of_bytes; -} - diff --git a/src/plugins/output/forwarder/src/ConnectionParams.h b/src/plugins/output/forwarder/src/ConnectionParams.h deleted file mode 100644 index 5f9fbfa7..00000000 --- a/src/plugins/output/forwarder/src/ConnectionParams.h +++ /dev/null @@ -1,136 +0,0 @@ -/** - * \file src/plugins/output/forwarder/src/ConnectionParams.h - * \author Michal Sedlak - * \brief Parameters for estabilishing connection - * \date 2021 - */ - -/* Copyright (C) 2021 CESNET, z.s.p.o. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * 3. Neither the name of the Company nor the names of its contributors - * may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * ALTERNATIVELY, provided that this notice is retained in full, this - * product may be distributed under the terms of the GNU General Public - * License (GPL) version 2 or later, in which case the provisions - * of the GPL apply INSTEAD OF those given above. - * - * This software is provided ``as is'', and any express or implied - * warranties, including, but not limited to, the implied warranties of - * merchantability and fitness for a particular purpose are disclaimed. - * In no event shall the company or contributors be liable for any - * direct, indirect, incidental, special, exemplary, or consequential - * damages (including, but not limited to, procurement of substitute - * goods or services; loss of use, data, or profits; or business - * interruption) however caused and on any theory of liability, whether - * in contract, strict liability, or tort (including negligence or - * otherwise) arising in any way out of the use of this software, even - * if advised of the possibility of such damage. - * - */ - -#pragma once - -#include -#include - -#include -#include -#include -#include -#include - -using unique_addrinfo = std::unique_ptr; - -enum class TransProto { Tcp, Udp }; - -struct ConnectionParams -{ - ConnectionParams(std::string address, std::string port, TransProto protocol) - : address(address) - , port(port) - , protocol(protocol) - { - } - - unique_addrinfo - resolve_address() - { - addrinfo *info; - addrinfo hints = {}; - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = (protocol == TransProto::Tcp ? SOCK_STREAM : SOCK_DGRAM); - hints.ai_protocol = (protocol == TransProto::Tcp ? IPPROTO_TCP : IPPROTO_UDP); - if (getaddrinfo(address.c_str(), port.c_str(), &hints, &info) == 0) { - return unique_addrinfo(info, &freeaddrinfo); - } else { - return unique_addrinfo(NULL, &freeaddrinfo); - } - } - - int - make_socket() - { - auto address_info = resolve_address(); - if (!address_info) { - return -1; - } - - int sockfd; - addrinfo *p; - - for (p = address_info.get(); p != NULL; p = p->ai_next) { - sockfd = socket(p->ai_family, p->ai_socktype, p->ai_protocol); - if (sockfd < 0) { - continue; - } - - if (protocol == TransProto::Udp) { - sockaddr_in sa = {}; - sa.sin_family = AF_INET; - sa.sin_port = 0; - sa.sin_addr.s_addr = INADDR_ANY; - sa.sin_port = 0; - if (bind(sockfd, (sockaddr *)&sa, sizeof(sa)) != 0) { - close(sockfd); - continue; - } - } - - if (connect(sockfd, p->ai_addr, p->ai_addrlen) != 0) { - close(sockfd); - continue; - } - - break; - } - - if (!p) { - return -1; - } - - return sockfd; - } - - std::string - str() - { - return std::string(protocol == TransProto::Tcp ? "TCP" : "UDP") + ":" - + address + ":" - + port; - } - - std::string address; - std::string port; - TransProto protocol; -}; \ No newline at end of file diff --git a/src/plugins/output/forwarder/src/Forwarder.cpp b/src/plugins/output/forwarder/src/Forwarder.cpp new file mode 100644 index 00000000..da0e89f1 --- /dev/null +++ b/src/plugins/output/forwarder/src/Forwarder.cpp @@ -0,0 +1,133 @@ +/** + * \file src/plugins/output/forwarder/src/Forwarder.cpp + * \author Michal Sedlak + * \brief Forwarder class implementation + * \date 2021 + */ + +/* Copyright (C) 2021 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include "Forwarder.h" + +Forwarder::Forwarder(Config config, ipx_ctx_t *log_ctx) : + m_config(config), + m_log_ctx(log_ctx) +{ + // Set up connector + std::vector con_params; + for (const auto &host_config : m_config.hosts) { + con_params.push_back(ConnectionParams{host_config.address, host_config.port, m_config.protocol}); + } + + m_connector.reset(new Connector(con_params, m_config.nb_premade_connections, + m_config.reconnect_secs, m_log_ctx)); + + // Set up hosts + for (const auto &host_config : m_config.hosts) { + m_hosts.emplace_back( + new Host(host_config.name, + ConnectionParams{host_config.address, host_config.port, m_config.protocol}, + m_log_ctx, + m_config.tmplts_resend_pkts, + m_config.tmplts_resend_secs, + m_config.forward_mode == ForwardMode::SENDTOALL, + *m_connector.get())); + + } +} + +void Forwarder::handle_session_message(ipx_msg_session_t *msg) +{ + const ipx_session *session = ipx_msg_session_get_session(msg); + + switch (ipx_msg_session_get_event(msg)) { + case IPX_MSG_SESSION_OPEN: + IPX_CTX_DEBUG(m_log_ctx, "New session %s", session->ident); + for (auto &host : m_hosts) { + host->setup_connection(session); + } + break; + + case IPX_MSG_SESSION_CLOSE: + IPX_CTX_DEBUG(m_log_ctx, "Closing session %s", session->ident); + for (auto &host : m_hosts) { + host->finish_connection(session); + } + break; + } +} + +void Forwarder::handle_ipfix_message(ipx_msg_ipfix_t *msg) +{ + // Forward message + switch (m_config.forward_mode) { + case ForwardMode::SENDTOALL: + forward_to_all(msg); + break; + + case ForwardMode::ROUNDROBIN: + forward_round_robin(msg); + break; + + default: assert(0); + } +} + +void +Forwarder::forward_to_all(ipx_msg_ipfix_t *msg) +{ + for (auto &host : m_hosts) { + host->forward_message(msg); + } +} + +void +Forwarder::forward_round_robin(ipx_msg_ipfix_t *msg) +{ + bool ok = false; + + for (size_t i = 0; i < m_hosts.size(); i++) { + auto &host = m_hosts[m_rr_index]; + ok = host->forward_message(msg); + m_rr_index = (m_rr_index + 1) % m_hosts.size(); + if (ok) { + break; + } + } + + if (!ok) { + IPX_CTX_WARNING(m_log_ctx, "Couldn't forward to any of the hosts, dropping message!", 0); + } +} diff --git a/src/plugins/output/forwarder/src/Forwarder.h b/src/plugins/output/forwarder/src/Forwarder.h index d75e6182..2fc11ddd 100644 --- a/src/plugins/output/forwarder/src/Forwarder.h +++ b/src/plugins/output/forwarder/src/Forwarder.h @@ -1,7 +1,7 @@ /** * \file src/plugins/output/forwarder/src/Forwarder.h * \author Michal Sedlak - * \brief Forwarder logic + * \brief Forwarder class header * \date 2021 */ @@ -41,402 +41,64 @@ #pragma once -#include "ConnectionManager.h" -#include "IPFIXMessage.h" -#include "MessageBuilder.h" +#include "Config.h" -#include -#include - -#include +#include #include -enum class ForwardMode { SendToAll, RoundRobin }; - -struct Session; -struct Client; - -struct Odid -{ - Odid() {} // Default constructor needed because of std::map - - Odid(Session &session, uint32_t odid) - : session(&session) - , odid(odid) - {} - - Session *session; - uint32_t odid; - uint32_t seq_num = 0; - - const fds_tsnapshot_t *templates_snapshot = NULL; - std::time_t last_templates_send_time = 0; - unsigned bytes_since_templates_sent = 0; - - std::string - str(); - - void - reset_values() - { - seq_num = 0; - templates_snapshot = NULL; - last_templates_send_time = 0; - bytes_since_templates_sent = 0; - } -}; - -struct Session -{ - Session(Connection &connection, Client &client, std::string ident) - : connection(connection) - , client(client) - , ident(ident) - {} - - Connection &connection; - Client &client; - std::string ident; - - std::map odids; - - std::string - str(); -}; - -struct Client -{ - Client(ConnectionManager &connection_manager, ConnectionParams connection_params, std::string name = "") - : connection_manager(connection_manager) - , connection_params(connection_params) - , name(name) - {} - - ConnectionManager &connection_manager; - ConnectionParams connection_params; - std::string name; - std::map> sessions; - - std::string - str() - { - return name; - } -}; - -std::string -Odid::str() -{ - return session->ident + "(" + std::to_string(odid) + ") -> " + session->client.name; -} - -std::string -Session::str() -{ - return ident + " -> " + client.name; -} +#include "Host.h" +#include "common.h" +#include "connector/Connector.h" -class Forwarder -{ +/// A class representing the forwarder itself +class Forwarder { public: - Forwarder(ipx_ctx_t *log_ctx) - : log_ctx(log_ctx) - {} - - void - set_transport_protocol(TransProto transport_protocol) - { - this->transport_protocol = transport_protocol; - } - - void - set_forward_mode(ForwardMode forward_mode) - { - this->forward_mode = forward_mode; - } - - void - set_connection_buffer_size(long number_of_bytes) - { - connection_manager.set_connection_buffer_size(number_of_bytes); - } - - void - set_template_refresh_interval_secs(int number_of_seconds) - { - this->template_refresh_interval_secs = number_of_seconds; - } - - void - set_template_refresh_interval_bytes(int number_of_bytes) - { - this->template_refresh_interval_bytes = number_of_bytes; - } - - void - set_reconnect_interval(int secs) - { - connection_manager.set_reconnect_interval(secs); - } - - void - add_client(std::string address, std::string port, std::string name = "") - { - auto connection_params = ConnectionParams { address, port, transport_protocol }; - if (!connection_params.resolve_address()) { - throw "Cannot resolve address " + address; - } - if (name.empty()) { - name = connection_params.str(); - } - auto client = Client { connection_manager, connection_params, name }; - clients.push_back(std::move(client)); - IPX_CTX_INFO(log_ctx, "Added client %s @ %s\n", name.c_str(), connection_params.str().c_str()); - } - - void - on_session_message(ipx_msg_session_t *session_msg) - { - const ipx_session *session = ipx_msg_session_get_session(session_msg); - switch (ipx_msg_session_get_event(session_msg)) { - case IPX_MSG_SESSION_OPEN: - for (auto &client : clients) { - open_session(client, session); - } - break; - case IPX_MSG_SESSION_CLOSE: - for (auto &client : clients) { - close_session(client, session); - } - break; - } - } - - void - on_ipfix_message(ipx_msg_ipfix_t *ipfix_msg) - { - auto message = IPFIXMessage { ipfix_msg }; - switch (forward_mode) { - case ForwardMode::RoundRobin: - forward_round_robin(message); - break; - case ForwardMode::SendToAll: - forward_to_all(message); - break; - } - } - - void - start() - { - connection_manager.start(); - } - - void - stop() - { - connection_manager.stop(); - - IPX_CTX_INFO(log_ctx, "Total bytes forwarded: %ld", total_bytes); - IPX_CTX_INFO(log_ctx, "Dropped messages: %ld", dropped_messages); - IPX_CTX_INFO(log_ctx, "Dropped data records: %ld", dropped_data_records); - } - - ~Forwarder() - { - } + /** + * \brief The constructor + * \param config The forwarder configuration + * \param log_ctx The logging context + */ + Forwarder(Config config, ipx_ctx_t *log_ctx); + + /** + * Disable copy and move constructors + */ + Forwarder(const Forwarder &) = delete; + Forwarder(Forwarder &&) = delete; + + /** + * \brief Handle a session message + * \param msg The session message + */ + void + handle_session_message(ipx_msg_session_t *msg); + + /** + * \brief Handle an IPFIX message + * \param msg The IPFIX message + */ + void + handle_ipfix_message(ipx_msg_ipfix_t *msg); + + /** + * \brief The destructor - finalize the forwarder + */ + ~Forwarder() {} private: - /// Logging context - ipx_ctx_t *log_ctx; + Config m_config; - /// Configuration - TransProto transport_protocol = TransProto::Tcp; - ForwardMode forward_mode = ForwardMode::SendToAll; - int template_refresh_interval_secs = 0; - int template_refresh_interval_bytes = 0; + ipx_ctx_t *m_log_ctx; - /// Mutating state - ConnectionManager connection_manager; - std::vector clients; - int rr_next_client = 0; + std::vector> m_hosts; - /// Statistics - long dropped_messages = 0; - long dropped_data_records = 0; - long total_bytes = 0; + size_t m_rr_index = 0; - void - open_session(Client &client, const ipx_session *session_info) - { - auto &connection = connection_manager.add_client(client.connection_params); - auto session_ptr = std::unique_ptr(new Session { connection, client, session_info->ident }); - IPX_CTX_INFO(log_ctx, "Opened session %s", session_ptr->str().c_str()); - client.sessions[session_info] = std::move(session_ptr); - } + std::unique_ptr m_connector; void - close_session(Client &client, const ipx_session *session_info) - { - auto &session = *client.sessions[session_info]; - session.connection.close(); - IPX_CTX_INFO(log_ctx, "Closed session %s", session.str().c_str()); - client.sessions.erase(session_info); - } + forward_to_all(ipx_msg_ipfix_t *msg); void - forward_to_all(IPFIXMessage &message) - { - for (auto &client : clients) { - if (!forward_message(client, message)) { - dropped_messages += 1; - dropped_data_records += message.drec_count(); - } - } - } - - void - forward_round_robin(IPFIXMessage &message) - { - int i = 0; - while (1) { - auto &client = next_client(); - if (forward_message(client, message)) { - break; - } - - i++; - - // If we went through all the clients multiple times in a row and all the buffers are still full, - // let's just give up and move onto the next message. If we loop for too long we'll start losing messages! - if (i < (int)clients.size() * 10) { - dropped_messages += 1; - dropped_data_records += message.drec_count(); - break; - } - } - } - - /// Pick the next client in round-robin mode - Client & - next_client() - { - if (rr_next_client == (int)clients.size()) { - rr_next_client = 0; - } - return clients[rr_next_client++]; - } - - /// Send all templates from the templates snapshot obtained from the message - /// through the session connection and update the state accordingly - /// - /// \return true if there was enough space in the connection buffer, false otherwise - bool - send_templates(Session &session, Odid &odid, IPFIXMessage &message) - { - auto templates_snapshot = message.get_templates_snapshot(); - - auto header = *message.header(); - header.seq_num = htonl(odid.seq_num); - - MessageBuilder builder; - builder.begin_message(header); - - fds_tsnapshot_for(templates_snapshot, - [](const fds_template *tmplt, void *data) -> bool { - auto &builder = *(MessageBuilder *)data; - builder.write_template(tmplt); - return true; - }, &builder); - - builder.finalize_message(); - - auto lock = session.connection.begin_write(); - if (builder.message_length() > session.connection.writeable()) { - // IPX_CTX_WARNING(log_ctx, - // "[%s] Cannot send templates because buffer is full! (need %dB, have %ldB)", - // odid.str().c_str(), builder.message_length(), session.connection.writeable()); - return false; - } - session.connection.write(builder.message_data(), builder.message_length()); - session.connection.commit_write(); - - odid.templates_snapshot = templates_snapshot; - odid.bytes_since_templates_sent = 0; - odid.last_templates_send_time = std::time(NULL); - - total_bytes += builder.message_length(); - // IPX_CTX_INFO(log_ctx, "[%s] Sent templates", odid.str().c_str()); - - return true; - } - - bool - should_refresh_templates(Odid &odid) - { - if (transport_protocol != TransProto::Udp) { - return false; - } - auto time_since = (std::time(NULL) - odid.last_templates_send_time); - return (time_since > (unsigned)template_refresh_interval_secs) - || (odid.bytes_since_templates_sent > (unsigned)template_refresh_interval_bytes); - } - - bool - templates_changed(Odid &odid, IPFIXMessage &message) - { - auto templates_snapshot = message.get_templates_snapshot(); - return templates_snapshot && odid.templates_snapshot != templates_snapshot; - } - - /// Forward message to the client, including templates update if needed - /// - /// \return true if there was enough space in the connection buffer, false otherwise - bool - forward_message(Client &client, IPFIXMessage &message) - { - auto &session = *client.sessions[message.session()]; - - if (session.connection.connection_lost_flag) { - for (auto &p : session.odids) { - p.second.reset_values(); - } - session.connection.connection_lost_flag = false; - } - - if (session.odids.find(message.odid()) == session.odids.end()) { - session.odids[message.odid()] = Odid { session, message.odid() }; - IPX_CTX_INFO(log_ctx, "[%s] Seen new ODID %u", session.str().c_str(), message.odid()); - } - auto &odid = session.odids[message.odid()]; - - if (should_refresh_templates(odid) || templates_changed(odid, message)) { - if (message.get_templates_snapshot() != nullptr && !send_templates(session, odid, message)) { - return false; - } - } - - auto lock = session.connection.begin_write(); - if (message.length() > session.connection.writeable()) { - // IPX_CTX_WARNING(log_ctx, - // "[%s] Cannot forward message because buffer is full! (need %dB, have %ldB)", - // odid.str().c_str(), message.length(), session.connection.writeable()); - return false; - } - - auto header = *message.header(); - header.seq_num = htonl(odid.seq_num); - session.connection.write(&header, sizeof(header)); - session.connection.write(message.data() + sizeof(header), message.length() - sizeof(header)); - session.connection.commit_write(); - - // IPX_CTX_DEBUG(log_ctx, "[%s] Forwarded message", odid.str().c_str()); - - odid.bytes_since_templates_sent += message.length(); - odid.seq_num += message.drec_count(); - - total_bytes += message.length(); - - return true; - } + forward_round_robin(ipx_msg_ipfix_t *msg); }; \ No newline at end of file diff --git a/src/plugins/output/forwarder/src/Host.cpp b/src/plugins/output/forwarder/src/Host.cpp new file mode 100644 index 00000000..26851e7e --- /dev/null +++ b/src/plugins/output/forwarder/src/Host.cpp @@ -0,0 +1,168 @@ +/** + * \file src/plugins/output/forwarder/src/Host.cpp + * \author Michal Sedlak + * \brief Host class implementation + * \date 2021 + */ + +/* Copyright (C) 2021 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include "Host.h" +#include + +Host::Host(const std::string &ident, ConnectionParams con_params, ipx_ctx_t *log_ctx, + unsigned int tmplts_resend_pkts, unsigned int tmplts_resend_secs, bool indicate_lost_msgs, + Connector &connector) : + m_ident(ident), + m_con_params(con_params), + m_log_ctx(log_ctx), + m_tmplts_resend_pkts(tmplts_resend_pkts), + m_tmplts_resend_secs(tmplts_resend_secs), + m_indicate_lost_msgs(indicate_lost_msgs), + m_connector(connector) +{ +} + +void +Host::setup_connection(const ipx_session *session) +{ + // There shouldn't be a connection for the session yet + assert(m_session_to_connection.find(session) == m_session_to_connection.end()); + + // Create new connection + IPX_CTX_INFO(m_log_ctx, "Setting up new connection to %s", m_ident.c_str()); + + m_session_to_connection.emplace(session, + std::unique_ptr(new Connection( + m_ident, + m_con_params, + m_log_ctx, + m_tmplts_resend_pkts, + m_tmplts_resend_secs, + m_connector))); + m_session_to_connection[session]->connect(); +} + +void +Host::finish_connection(const ipx_session *session) +{ + IPX_CTX_INFO(m_log_ctx, "Finishing a connection to %s", m_ident.c_str()); + + // Get the corresponding connection and remove it from the session map + std::unique_ptr &connection = m_session_to_connection[session]; + assert(connection); + + // Try to send the waiting transfers if there are any, if the connection is dropped just finish anyway + if (connection->check_connected()) { + + try { + connection->advance_transfers(); + + } catch (const ConnectionError &) { + // Ignore... + } + } + + if (connection->waiting_transfers_cnt() > 0) { + IPX_CTX_WARNING(m_log_ctx, "Dropping %zu transfers when finishing connection", + connection->waiting_transfers_cnt()); + } + + IPX_CTX_INFO(m_log_ctx, "Connection to %s finished", m_ident.c_str()); + + m_session_to_connection.erase(session); +} + +bool +Host::forward_message(ipx_msg_ipfix_t *msg) +{ + const ipx_session *session = ipx_msg_ipfix_get_ctx(msg)->session; + Connection &connection = *m_session_to_connection[session].get(); + + if (!connection.check_connected()) { + if (m_indicate_lost_msgs) { + connection.lose_message(msg); + } + return false; + } + + try { + connection.advance_transfers(); + + if (connection.waiting_transfers_cnt() > 0) { + IPX_CTX_DEBUG(m_log_ctx, "Message to %s not forwarded because there are unsent transfers\n", m_ident.c_str()); + if (m_indicate_lost_msgs) { + connection.lose_message(msg); + } + return false; + } + + IPX_CTX_DEBUG(m_log_ctx, "Forwarding message to %s\n", m_ident.c_str()); + + connection.forward_message(msg); + + } catch (const ConnectionError &err) { + IPX_CTX_ERROR(m_log_ctx, "Lost connection while forwarding: %s", err.what()); + connection.connect(); + return false; + } + + return true; +} + +Host::~Host() +{ + for (auto &p : m_session_to_connection) { + std::unique_ptr &connection = p.second; + + // Try to send the waiting transfers if there are any, if the connection is dropped just finish anyway + if (connection->check_connected()) { + + try { + connection->advance_transfers(); + + } catch (const ConnectionError &) { + // Ignore... + } + } + + if (connection->waiting_transfers_cnt() > 0) { + IPX_CTX_WARNING(m_log_ctx, "Dropping %zu transfers when closing connection %s", + connection->waiting_transfers_cnt(), connection->ident().c_str()); + } + } + + IPX_CTX_INFO(m_log_ctx, "All connections to %s closed", m_ident.c_str()); +} diff --git a/src/plugins/output/forwarder/src/Host.h b/src/plugins/output/forwarder/src/Host.h new file mode 100644 index 00000000..4189f566 --- /dev/null +++ b/src/plugins/output/forwarder/src/Host.h @@ -0,0 +1,119 @@ +/** + * \file src/plugins/output/forwarder/src/Host.h + * \author Michal Sedlak + * \brief Host class header + * \date 2021 + */ + +/* Copyright (C) 2021 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#pragma once + +#include +#include +#include +#include "common.h" +#include "Config.h" +#include "Connection.h" +#include "connector/Connector.h" + +/// A class representing one of the subcollectors messages are forwarded to +class Host { +public: + /** + * \brief The constructor + * \param ident The host identification + * \param con_params The connection parameters + * \param log_ctx The logging context + * \param tmplts_resend_pkts Interval in packets after which templates are resend (UDP only) + * \param tmplts_resend_secs Interval in seconds after which templates are resend (UDP only) + * \param indicate_lost_msgs Indicate that the message has been lost if it couldn't be forwarded + * by increasing the sequence numbers + */ + Host(const std::string &ident, ConnectionParams con_params, ipx_ctx_t *log_ctx, + unsigned int tmplts_resend_pkts, unsigned int tmplts_resend_secs, bool indicate_lost_msgs, + Connector &connector); + + /** + * Disable copy and move constructors + */ + Host(const Host &) = delete; + Host(Host &&) = delete; + + /** + * \brief The destructor - finishes all the connections + */ + ~Host(); + + /** + * \brief Set up a new connection for the sesison + * \param session The session + */ + void + setup_connection(const ipx_session *session); + + /** + * \brief Finish a connection for the sesison + * \param session The session + */ + void + finish_connection(const ipx_session *session); + + /** + * \brief Forward an IPFIX message to this host + * \param msg The IPFIX message + * \return true on success, false on failure + * \throw ConnectionError when the connection fails + */ + bool + forward_message(ipx_msg_ipfix_t *msg); + +private: + const std::string &m_ident; + + ConnectionParams m_con_params; + + ipx_ctx_t *m_log_ctx; + + unsigned int m_tmplts_resend_pkts; + + unsigned int m_tmplts_resend_secs; + + bool m_indicate_lost_msgs; + + Connector &m_connector; + + std::unordered_map> m_session_to_connection; +}; \ No newline at end of file diff --git a/src/plugins/output/forwarder/src/Message.cpp b/src/plugins/output/forwarder/src/Message.cpp new file mode 100644 index 00000000..08229dd7 --- /dev/null +++ b/src/plugins/output/forwarder/src/Message.cpp @@ -0,0 +1,185 @@ +/** + * \file src/plugins/output/forwarder/src/Message.cpp + * \author Michal Sedlak + * \brief Message class implementation + * \date 2021 + */ + +/* Copyright (C) 2021 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include "Message.h" +#include + +static uint16_t +get_set_id(fds_template_type template_type) +{ + switch (template_type) { + case FDS_TYPE_TEMPLATE: + return FDS_IPFIX_SET_TMPLT; + + case FDS_TYPE_TEMPLATE_OPTS: + return FDS_IPFIX_SET_OPTS_TMPLT; + + default: assert(0); + } +} + +void +Message::add_set(const fds_ipfix_set_hdr *set) +{ + if (m_current_set_hdr) { + finalize_set(); + } + + uint16_t set_len = ntohs(set->length); + add_part((uint8_t *) set, set_len); + m_current_set_hdr = nullptr; +} + +void +Message::add_template(const fds_template *tmplt) +{ + require_set(get_set_id(tmplt->type)); + write(tmplt->raw.data, tmplt->raw.length); + m_current_set_hdr->length += tmplt->raw.length; +} + +void +Message::add_template_withdrawal_all() +{ + finalize_set(); + + require_set(FDS_IPFIX_SET_TMPLT); + uint16_t wdrl2[2] = { htons(FDS_IPFIX_SET_TMPLT), 0 }; + write(&wdrl2); + m_current_set_hdr->length += sizeof(wdrl2); + finalize_set(); + + require_set(FDS_IPFIX_SET_OPTS_TMPLT); + uint16_t wdrl3[2] = { htons(FDS_IPFIX_SET_OPTS_TMPLT), 0 }; + write(&wdrl3); + m_current_set_hdr->length += sizeof(wdrl3); + finalize_set(); +} + +void +Message::finalize() +{ + finalize_set(); + m_msg_hdr->length = htons(m_length); +} + +void +Message::start(const fds_ipfix_msg_hdr *msg_hdr) +{ + m_parts.clear(); + m_length = 0; + m_buffer_pos = 0; + m_last_part_from_buffer = false; + m_msg_hdr = nullptr; + m_current_set_hdr = nullptr; + + m_msg_hdr = write(msg_hdr); +} + +void +Message::add_part(uint8_t *data, uint16_t length) +{ + assert(data < m_buffer || data >= m_buffer + BUFFER_SIZE); + + iovec part; + part.iov_base = data; + part.iov_len = length; + m_parts.push_back(part); + + m_length += length; + + m_last_part_from_buffer = false; +} + +uint8_t * +Message::write(const uint8_t *data, uint16_t length) +{ + assert(BUFFER_SIZE - m_buffer_pos >= length); + + uint8_t *p = &m_buffer[m_buffer_pos]; + m_buffer_pos += length; + memcpy(p, data, length); + + // If the last part is also allocated from our buffer, we can just expand it rather than pushing a new part + if (m_last_part_from_buffer) { + m_parts.back().iov_len += length; + + } else { + iovec part; + part.iov_base = p; + part.iov_len = length; + m_parts.push_back(part); + + m_last_part_from_buffer = true; + } + + m_length += length; + + return p; +} + +void +Message::require_set(uint16_t set_id) +{ + if (!m_current_set_hdr || m_current_set_hdr->flowset_id != set_id) { + finalize_set(); + + fds_ipfix_set_hdr hdr; + hdr.flowset_id = set_id; + hdr.length = sizeof(fds_ipfix_set_hdr); + m_current_set_hdr = write(&hdr); + } +} + +void +Message::finalize_set() +{ + // If there is no set, do nothing + if (!m_current_set_hdr) { + return; + } + + assert(m_current_set_hdr->length > sizeof(fds_ipfix_set_hdr)); + + m_current_set_hdr->flowset_id = htons(m_current_set_hdr->flowset_id); + m_current_set_hdr->length = htons(m_current_set_hdr->length); + m_current_set_hdr = nullptr; +} \ No newline at end of file diff --git a/src/plugins/output/forwarder/src/Message.h b/src/plugins/output/forwarder/src/Message.h new file mode 100644 index 00000000..890808c6 --- /dev/null +++ b/src/plugins/output/forwarder/src/Message.h @@ -0,0 +1,169 @@ +/** + * \file src/plugins/output/forwarder/src/Message.h + * \author Michal Sedlak + * \brief Message class header + * \date 2021 + */ + +/* Copyright (C) 2021 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#pragma once + +#include +#include +#include + +/// A class for building IPFIX messages +class Message { +public: + /** + * \brief The default constructor + */ + Message() {} + + /// Do not allow moving or copying as the parts vector holds addresses to the buffer within the instance + Message(const Message &) = delete; + Message(Message &&) = delete; + + /** + * \brief Start new message, any existing message data is cleared + * \param msg_hdr The new message header + */ + void + start(const fds_ipfix_msg_hdr *msg_hdr); + + /** + * \brief Add an IPFIX set + * \param set Pointer to the set + * \warning No data is copied, the pointer is stored directly + */ + void + add_set(const fds_ipfix_set_hdr *set); + + /** + * \brief Add a template + * \param tmplt The template + * \note Unlike add_set, the template data is copied and stored in an internal buffer + */ + void + add_template(const fds_template *tmplt); + + /** + * \brief Add a template withdrawal + * \param tmplt The template to withdraw + */ + void + add_template_withdrawal(const fds_template *tmplt); + + /** + * \brief Add a template withdrawal all + */ + void + add_template_withdrawal_all(); + + /** + * \brief Finalize the message + * \warning This HAS to be called AFTER everything was added to the message and BEFORE parts() are accessed. + */ + void + finalize(); + + /** + * \brief Access the message parts + * \return The message parts + */ + std::vector &parts() { return m_parts; } + + /** + * \brief Get the total length of the message + * \return The length + */ + uint16_t length() const { return m_length; } + + /** + * \brief Check if the message is empty or only contains headers but no content + * \return true or false + */ + bool empty() const { return length() <= sizeof(fds_ipfix_msg_hdr) + sizeof(fds_ipfix_set_hdr); } + + /** + * \brief Access the message header + * \return The message header + */ + const fds_ipfix_msg_hdr *header() const { return m_msg_hdr; } + +private: + static constexpr uint16_t BUFFER_SIZE = UINT16_MAX; + + std::vector m_parts; + + uint16_t m_length = 0; + + uint8_t m_buffer[BUFFER_SIZE]; //NOTE: Maybe this should be dynamically expanding instead? + + uint16_t m_buffer_pos = 0; + + fds_ipfix_msg_hdr *m_msg_hdr = nullptr; + + fds_ipfix_set_hdr *m_current_set_hdr = nullptr; + + bool m_last_part_from_buffer = false; + + void + add_part(uint8_t *data, uint16_t length); + + template + T * + write(const T *item); + + uint8_t * + write(const uint8_t *data, uint16_t length); + + void + require_set(uint16_t set_id); + + void + finalize_set(); +}; + + +template +T * +Message::write(const T *item) +{ + T *p = (T *) write((const uint8_t *) item, sizeof(T)); + return p; +} + diff --git a/src/plugins/output/forwarder/src/MessageBuilder.h b/src/plugins/output/forwarder/src/MessageBuilder.h deleted file mode 100644 index 941dfd9a..00000000 --- a/src/plugins/output/forwarder/src/MessageBuilder.h +++ /dev/null @@ -1,150 +0,0 @@ -/** - * \file src/plugins/output/forwarder/src/MessageBuilder.h - * \author Michal Sedlak - * \brief IPFIX message builder - * \date 2021 - */ - -/* Copyright (C) 2021 CESNET, z.s.p.o. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * 3. Neither the name of the Company nor the names of its contributors - * may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * ALTERNATIVELY, provided that this notice is retained in full, this - * product may be distributed under the terms of the GNU General Public - * License (GPL) version 2 or later, in which case the provisions - * of the GPL apply INSTEAD OF those given above. - * - * This software is provided ``as is'', and any express or implied - * warranties, including, but not limited to, the implied warranties of - * merchantability and fitness for a particular purpose are disclaimed. - * In no event shall the company or contributors be liable for any - * direct, indirect, incidental, special, exemplary, or consequential - * damages (including, but not limited to, procurement of substitute - * goods or services; loss of use, data, or profits; or business - * interruption) however caused and on any theory of liability, whether - * in contract, strict liability, or tort (including negligence or - * otherwise) arising in any way out of the use of this software, even - * if advised of the possibility of such damage. - * - */ - -#pragma once - -#include -#include - -#include -#include - -class MessageBuilder -{ -public: - void - begin_message(fds_ipfix_msg_hdr message_header) - { - write(&message_header, sizeof(message_header)); - } - - void - write_template(const fds_template *tmplt) - { - if (current_set_id != get_template_set_id(tmplt->type)) { - end_template_set(); - begin_template_set(tmplt->type); - } - - write(tmplt->raw.data, tmplt->raw.length); - current_set_length += tmplt->raw.length; - } - - void - finalize_message() - { - end_template_set(); - header()->length = htons(write_offset); - } - - uint8_t * - message_data() - { - return &buffer[0]; - } - - int - message_length() - { - return write_offset; - } - -private: - std::vector buffer; - int write_offset = 0; - int set_header_offset = -1; - int current_set_id = 0; - int current_set_length = 0; - - fds_ipfix_msg_hdr * - header() - { - return (fds_ipfix_msg_hdr *)&buffer[0]; - } - - fds_ipfix_set_hdr * - current_set_header() - { - return (fds_ipfix_set_hdr *)&buffer[set_header_offset]; - } - - void - write(void *data, int length) - { - if (((int)buffer.size() - write_offset) < length) { - buffer.resize(buffer.size() + 1024); - return write(data, length); - } - std::memcpy(&buffer[write_offset], data, length); - write_offset += length; - } - - uint16_t - get_template_set_id(fds_template_type template_type) - { - return (template_type == FDS_TYPE_TEMPLATE - ? FDS_IPFIX_SET_TMPLT : FDS_IPFIX_SET_OPTS_TMPLT); - } - - void - begin_template_set(fds_template_type template_type) - { - fds_ipfix_set_hdr set_header = {}; - set_header.flowset_id = htons(get_template_set_id(template_type)); - - set_header_offset = write_offset; - write(&set_header, sizeof(set_header)); - - current_set_length = sizeof(set_header); - current_set_id = get_template_set_id(template_type); - } - - void - end_template_set() - { - if (set_header_offset != -1) { - current_set_header()->length = htons(current_set_length); - } - current_set_id = 0; - current_set_length = 0; - set_header_offset = -1; - } -}; \ No newline at end of file diff --git a/src/plugins/output/forwarder/src/Sender.cpp b/src/plugins/output/forwarder/src/Sender.cpp new file mode 100644 index 00000000..635da33c --- /dev/null +++ b/src/plugins/output/forwarder/src/Sender.cpp @@ -0,0 +1,253 @@ +/** + * \file src/plugins/output/forwarder/src/Sender.cpp + * \author Michal Sedlak + * \brief Sender class implementation + * \date 2021 + */ + +/* Copyright (C) 2021 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include "Sender.h" +#include + +class DrecsForwardIterator { +public: + DrecsForwardIterator(ipx_msg_ipfix_t *msg) : m_msg(msg) {} + + ipx_ipfix_record * + get_drec_after_set(fds_ipfix_set_hdr *set_hdr) + { + uint32_t drec_cnt = ipx_msg_ipfix_get_drec_cnt(m_msg); + uint8_t *set_end = (uint8_t *) set_hdr + ntohs(set_hdr->length); + + for (; m_idx < drec_cnt; m_idx++) { + ipx_ipfix_record *drec = ipx_msg_ipfix_get_drec(m_msg, m_idx); + if (drec->rec.data > set_end) { + return drec; + } + } + + return nullptr; + } + + ipx_ipfix_record * + get_drec_in_set(fds_ipfix_set_hdr *set_hdr) + { + uint32_t drec_cnt = ipx_msg_ipfix_get_drec_cnt(m_msg); + uint8_t *set_start = (uint8_t *) set_hdr; + uint8_t *set_end = (uint8_t *) set_hdr + ntohs(set_hdr->length); + + for (; m_idx < drec_cnt; m_idx++) { + ipx_ipfix_record *drec = ipx_msg_ipfix_get_drec(m_msg, m_idx); + if (drec->rec.data >= set_end) { + return nullptr; + } + if (drec->rec.data >= set_start) { + return drec; + } + } + + return nullptr; + } + + uint32_t idx() const { return m_idx; } + +private: + ipx_msg_ipfix_t *m_msg; + uint32_t m_idx = 0; +}; + +Sender::Sender(std::function emit_callback, bool do_withdrawals, + unsigned int tmplts_resend_pkts, unsigned int tmplts_resend_secs) : + m_emit_callback(emit_callback), + m_do_withdrawals(do_withdrawals), + m_tmplts_resend_pkts(tmplts_resend_pkts), + m_tmplts_resend_secs(tmplts_resend_secs) +{ +} + +void +Sender::process_message(ipx_msg_ipfix_t *msg) +{ + // Get current real time + timespec realtime_ts; + if (clock_gettime(CLOCK_REALTIME_COARSE, &realtime_ts) != 0) { + throw errno_runtime_error(errno, "clock_gettime"); + } + + // Begin the message, use the original message header with the sequence number and time replaced + fds_ipfix_msg_hdr msg_hdr = *(fds_ipfix_msg_hdr *) ipx_msg_ipfix_get_packet(msg); + msg_hdr.seq_num = htonl(m_seq_num); + msg_hdr.export_time = htonl(realtime_ts.tv_sec); + + m_message.start(&msg_hdr); + + // Get current time + time_t now = get_monotonic_time(); + + // Send templates update if necessary and possible + ipx_ipfix_record *drec = ipx_msg_ipfix_get_drec(msg, 0); + + if (drec) { + + const fds_tsnapshot_t *tsnap = drec->rec.snap; + + // If templates changed or any of the resend intervals elapsed + if (m_tsnap != tsnap + || (m_tmplts_resend_pkts != 0 && m_pkts_since_tmplts_sent >= m_tmplts_resend_pkts) + || (m_tmplts_resend_secs != 0 && now - m_last_tmplts_sent_time >= m_tmplts_resend_secs)) { + + process_templates(tsnap, m_seq_num); + } + } + + // Get sets from the message + ipx_ipfix_set *sets; + size_t num_sets; + ipx_msg_ipfix_get_sets(msg, &sets, &num_sets); + + // For walking through the parsed ipfix records of the message to get a parsed ipfix record + // belonging to a set to retieve e.g. a template snapshot from it + DrecsForwardIterator drecs_iter(msg); + + for (size_t i = 0; i < num_sets; i++) { + + fds_ipfix_set_hdr *set_hdr = sets[i].ptr; + uint16_t set_id = ntohs(set_hdr->flowset_id); + + // If it's not a template set, add the set as is + if (set_id != FDS_IPFIX_SET_TMPLT && set_id != FDS_IPFIX_SET_OPTS_TMPLT) { + + ipx_ipfix_record *drec = drecs_iter.get_drec_in_set(set_hdr); + + if (!drec) { + // No parsed data record belonging to the set means we don't have a template + // Skip the data set + continue; + } + + m_message.add_set(set_hdr); + continue; + } + + // It is a template set... + + // Find first data record after the template set + ipx_ipfix_record *drec = drecs_iter.get_drec_after_set(set_hdr); + + // The template set is at the end, we'll have to wait for next message to grab the template snapshot + if (!drec) { + break; + } + + // Get template snapshot from the data record after template set + const fds_tsnapshot_t *tsnap = drec->rec.snap; + + // In case the template set is at the start of the message and we already sent the templates + if (m_tsnap == tsnap) { + continue; + } + + // The next sequence number in case we'll need to start another message + uint32_t next_seq_num = m_seq_num + drecs_iter.idx(); + + process_templates(tsnap, next_seq_num); + } + + + if (!m_message.empty()) { + m_message.finalize(); + emit_message(); + } + + m_seq_num += ipx_msg_ipfix_get_drec_cnt(msg); + m_pkts_since_tmplts_sent++; +} + + +void +Sender::lose_message(ipx_msg_ipfix_t *msg) +{ + m_seq_num += ipx_msg_ipfix_get_drec_cnt(msg); +} + +void +Sender::process_templates(const fds_tsnapshot_t *tsnap, uint32_t next_seq_num) +{ + if (m_do_withdrawals) { + m_message.add_template_withdrawal_all(); + } + + tsnapshot_for_each(tsnap, [&](const fds_template *tmplt) { + // Should we start another message? + if (m_message.length() + sizeof(fds_ipfix_set_hdr) + tmplt->raw.length > TMPLTMSG_MAX_LENGTH && !m_message.empty()) { + m_message.finalize(); + + emit_message(); + + fds_ipfix_msg_hdr msg_hdr = *m_message.header(); + msg_hdr.seq_num = htonl(next_seq_num); + m_message.start(&msg_hdr); + } + + m_message.add_template(tmplt); + }); + + + if (!m_message.empty()) { + m_message.finalize(); + emit_message(); + } + + m_tsnap = tsnap; + m_last_tmplts_sent_time = get_monotonic_time(); + m_pkts_since_tmplts_sent = 0; + + fds_ipfix_msg_hdr msg_hdr = *m_message.header(); + msg_hdr.seq_num = htonl(next_seq_num); + m_message.start(&msg_hdr); +} + +void +Sender::emit_message() +{ + m_emit_callback(m_message); +} + +void +Sender::clear_templates() +{ + m_tsnap = nullptr; +} diff --git a/src/plugins/output/forwarder/src/Sender.h b/src/plugins/output/forwarder/src/Sender.h new file mode 100644 index 00000000..9df08d33 --- /dev/null +++ b/src/plugins/output/forwarder/src/Sender.h @@ -0,0 +1,110 @@ +/** + * \file src/plugins/output/forwarder/src/Sender.h + * \author Michal Sedlak + * \brief Sender class header + * \date 2021 + */ + +/* Copyright (C) 2021 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#pragma once + +#include +#include "Message.h" +#include "common.h" +#include + +constexpr size_t TMPLTMSG_MAX_LENGTH = 2500; //NOTE: Maybe this should be configurable from the XML? + +/// A class to emit the messages to be sent through the connection in the process of forwarding a message +/// Each connection contains one sender per ODID +class Sender { +public: + /** + * \brief The constructor + * \param emit_callback The callback to be called when the sender emits a message to be sent + * \param do_withdrawals Specifies whether template messages should include withdrawals + * \param tmplts_resend_pkts Interval in packets after which templates are resend (0 = never) + * \param tmplts_resend_secs Interval in seconds after which templates are resend (0 = never) + */ + Sender(std::function emit_callback, bool do_withdrawals, + unsigned int tmplts_resend_pkts, unsigned int tmplts_resend_secs); + + /** + * \brief Receive an IPFIX message and emit messages to be sent to the receiving host + * \param msg The IPFIX message + */ + void + process_message(ipx_msg_ipfix_t *msg); + + /** + * \brief Lose an IPFIX message, i.e. update the internal state as if it has been forwarded + * even though it is not being sent + * \param msg The IPFIX message + */ + void + lose_message(ipx_msg_ipfix_t *msg); + + /** + * \brief Clear the templates state, i.e. force the templates to resend the next round + */ + void + clear_templates(); + +private: + std::function m_emit_callback; + + bool m_do_withdrawals; + + unsigned int m_tmplts_resend_pkts; + + unsigned int m_tmplts_resend_secs; + + uint32_t m_seq_num = 0; + + const fds_tsnapshot_t *m_tsnap = nullptr; + + unsigned int m_pkts_since_tmplts_sent = 0; + + time_t m_last_tmplts_sent_time = 0; + + Message m_message; + + void + process_templates(const fds_tsnapshot_t *tsnap, uint32_t next_seq_num); + + void + emit_message(); +}; \ No newline at end of file diff --git a/src/plugins/output/forwarder/src/common.cpp b/src/plugins/output/forwarder/src/common.cpp new file mode 100644 index 00000000..100c435e --- /dev/null +++ b/src/plugins/output/forwarder/src/common.cpp @@ -0,0 +1,98 @@ +/** + * \file src/plugins/output/forwarder/src/common.cpp + * \author Michal Sedlak + * \brief Common use functions and structures + * \date 2021 + */ + +/* Copyright (C) 2021 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include "common.h" +#include +#include +#include +#include + +#include + +void +tsnapshot_for_each(const fds_tsnapshot_t *tsnap, std::function callback) +{ + struct CallbackData { + std::function callback; + std::exception_ptr ex; + }; + + CallbackData cb_data; + cb_data.callback = callback; + cb_data.ex = nullptr; + + fds_tsnapshot_for(tsnap, + [](const fds_template *tmplt, void *data) -> bool { + CallbackData &cb_data = *(CallbackData *) data; + + try { + cb_data.callback(tmplt); + return true; + + } catch (...) { + cb_data.ex = std::current_exception(); + return false; + } + + }, &cb_data); + + if (cb_data.ex) { + std::rethrow_exception(cb_data.ex); + } +} + +time_t +get_monotonic_time() +{ + timespec ts; + if (clock_gettime(CLOCK_MONOTONIC_COARSE, &ts) != 0) { + throw errno_runtime_error(errno, "clock_gettime"); + } + return ts.tv_sec; +} + +std::runtime_error +errno_runtime_error(int errno_, const std::string &func_name) +{ + char *errbuf; + ipx_strerror(errno_, errbuf); + return std::runtime_error(func_name + "() failed: " + std::string(errbuf)); +} diff --git a/src/plugins/output/forwarder/src/common.h b/src/plugins/output/forwarder/src/common.h new file mode 100644 index 00000000..48b2a23a --- /dev/null +++ b/src/plugins/output/forwarder/src/common.h @@ -0,0 +1,157 @@ +/** + * \file src/plugins/output/forwarder/src/common.h + * \author Michal Sedlak + * \brief Common use functions and structures + * \date 2021 + */ + +/* Copyright (C) 2021 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +enum class Protocol : uint8_t { + UNASSIGNED, + TCP, + UDP +}; + +/** + * \brief Connection parameters + */ +struct ConnectionParams { + /// The IP address or hostname + std::string address; + /// The port + uint16_t port; + /// The transport protocol + Protocol protocol; + + struct Hasher { + size_t + operator()(const ConnectionParams ¶m) const + { + return std::hash()(param.address) + ^ std::hash()(param.port) + ^ std::hash()(uint8_t(param.protocol)); + } + }; + + bool + operator==(const ConnectionParams &other) const + { + return address == other.address && port == other.port && protocol == other.protocol; + } +}; + +/** + * \brief RAII wrapper for file descriptor such as a socket fd + */ +class UniqueFd { +public: + UniqueFd() {} + + UniqueFd(int fd) : m_fd(fd) {} + + UniqueFd(const UniqueFd &) = delete; + + UniqueFd &operator=(const UniqueFd &) = delete; + + UniqueFd(UniqueFd &&other) { + close(); + std::swap(m_fd, other.m_fd); + } + + UniqueFd &operator=(UniqueFd &&other) { + if (this != &other) { + close(); + std::swap(m_fd, other.m_fd); + } + return *this; + } + + ~UniqueFd() { + close(); + } + + void reset(int fd = -1) { + close(); + m_fd = fd; + } + + int get() const { return m_fd; } + +private: + int m_fd = -1; + + void close() { + if (m_fd >= 0) { + ::close(m_fd); + m_fd = -1; + } + } +}; + +/** + * \brief A C++ wrapper for the fds_tsnapshot_for function that can handle exceptions + * \param tsnap The template snapshot + * \param callback The callback to be called for each of the templates in the snapshot + * \throw Whatever the callback might throw + */ +void +tsnapshot_for_each(const fds_tsnapshot_t *tsnap, std::function callback); + +/** + * \brief Get monotonic time to be used e.g. for calculating elapsed seconds + * \return The monotonic time + * \throw std::runtime_error on failure + */ +time_t +get_monotonic_time(); + +/** + * \brief Create runtime error from errno + * \param errno_ The errno + * \param func_name The function name to use in the error message + * \return The runtime error + */ +std::runtime_error +errno_runtime_error(int errno_, const std::string &func_name); diff --git a/src/plugins/output/forwarder/src/config.h b/src/plugins/output/forwarder/src/config.h deleted file mode 100644 index 81298548..00000000 --- a/src/plugins/output/forwarder/src/config.h +++ /dev/null @@ -1,306 +0,0 @@ -/** - * \file src/plugins/output/forwarder/src/config.h - * \author Michal Sedlak - * \brief Plugin configuration - * \date 2021 - */ - -/* Copyright (C) 2021 CESNET, z.s.p.o. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * 3. Neither the name of the Company nor the names of its contributors - * may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * ALTERNATIVELY, provided that this notice is retained in full, this - * product may be distributed under the terms of the GNU General Public - * License (GPL) version 2 or later, in which case the provisions - * of the GPL apply INSTEAD OF those given above. - * - * This software is provided ``as is'', and any express or implied - * warranties, including, but not limited to, the implied warranties of - * merchantability and fitness for a particular purpose are disclaimed. - * In no event shall the company or contributors be liable for any - * direct, indirect, incidental, special, exemplary, or consequential - * damages (including, but not limited to, procurement of substitute - * goods or services; loss of use, data, or profits; or business - * interruption) however caused and on any theory of liability, whether - * in contract, strict liability, or tort (including negligence or - * otherwise) arising in any way out of the use of this software, even - * if advised of the possibility of such damage. - * - */ - -#pragma once - -#include "Forwarder.h" - -#include -#include - -#include -#include - -/// DIY std::optional -template -class Maybe -{ -public: - Maybe() - : has_value_(false) - {} - - Maybe(T value) - : has_value_(true) - , value_(value) - {} - - bool - has_value() - { - return has_value_; - } - - T - value() - { - return value_; - } - -private: - bool has_value_; - T value_; -}; - - -/// Parses the XML config string and configures forwarder accordingly -void -parse_and_configure(ipx_ctx_t *log_ctx, const char *xml_config, Forwarder &forwarder) -{ - /// - /// Config schema definition - /// - enum { - MODE, - PROTOCOL, - RECONNECT_INTERVAL_SECS, - TEMPLATE_REFRESH_INTERVAL_SECS, - TEMPLATE_REFRESH_INTERVAL_BYTES, - CONNECTION_BUFFER_SIZE, - HOSTS, - HOST, - NAME, - ADDRESS, - PORT - }; - - fds_xml_args host_schema[] = { - FDS_OPTS_ELEM(NAME , "name" , FDS_OPTS_T_STRING, FDS_OPTS_P_OPT), - FDS_OPTS_ELEM(ADDRESS, "address", FDS_OPTS_T_STRING, 0 ), - FDS_OPTS_ELEM(PORT , "port" , FDS_OPTS_T_STRING, 0 ), - FDS_OPTS_END - }; - - fds_xml_args hosts_schema[] = { - FDS_OPTS_NESTED(HOST, "host", host_schema, FDS_OPTS_P_MULTI), - FDS_OPTS_END - }; - - fds_xml_args params_schema[] = { - FDS_OPTS_ROOT ("params"), - FDS_OPTS_ELEM (MODE , "mode" , FDS_OPTS_T_STRING, 0 ), - FDS_OPTS_ELEM (PROTOCOL , "protocol" , FDS_OPTS_T_STRING, 0 ), - FDS_OPTS_ELEM (CONNECTION_BUFFER_SIZE , "connectionBufferSize" , FDS_OPTS_T_INT , FDS_OPTS_P_OPT), - FDS_OPTS_ELEM (TEMPLATE_REFRESH_INTERVAL_SECS , "templateRefreshIntervalSecs" , FDS_OPTS_T_INT , FDS_OPTS_P_OPT), - FDS_OPTS_ELEM (TEMPLATE_REFRESH_INTERVAL_BYTES, "templateRefreshIntervalBytes", FDS_OPTS_T_INT , FDS_OPTS_P_OPT), - FDS_OPTS_ELEM (RECONNECT_INTERVAL_SECS , "reconnectIntervalSecs" , FDS_OPTS_T_INT , FDS_OPTS_P_OPT), - FDS_OPTS_NESTED(HOSTS , "hosts" , hosts_schema , 0 ), - FDS_OPTS_END - }; - - /// - /// Default parameter values - /// - const int default_template_refresh_interval_secs = 10 * 60; - const int default_template_refresh_interval_bytes = 5 * 1024 * 1024; - const int default_reconnect_interval_secs = 10; - const int default_connection_buffer_size = 4 * 1024 * 1024; - - /// - /// Parsed parameters - /// - struct HostInfo { - std::string name; - std::string address; - std::string port; - }; - - std::string mode; - std::string protocol; - std::vector hosts; - Maybe connection_buffer_size; - Maybe template_refresh_interval_secs; - Maybe template_refresh_interval_bytes; - Maybe reconnect_interval_secs; - - /// - /// Parsing - /// - auto parser = std::unique_ptr(fds_xml_create(), &fds_xml_destroy); - if (!parser) { - throw std::string("Cannot create XML parser"); - } - - auto lower = [](std::string s) { - for (auto &c : s) { - c = std::tolower(c); - } - return s; - }; - - auto parser_error = [&]() { - throw std::string("XML parser error: ") + fds_xml_last_err(parser.get()); - }; - - if (fds_xml_set_args(parser.get(), params_schema) != FDS_OK) { - parser_error(); - } - - auto params_elem = fds_xml_parse_mem(parser.get(), xml_config, true); - if (!params_elem) { - parser_error(); - } - - auto process_host = [&](fds_xml_ctx_t *host_elem) { - HostInfo host; - const fds_xml_cont *content; - while (fds_xml_next(host_elem, &content) != FDS_EOC) { - switch (content->id) { - case NAME: - host.name = std::string(content->ptr_string); - break; - case ADDRESS: - host.address = std::string(content->ptr_string); - break; - case PORT: - host.port = std::string(content->ptr_string); - break; - } - } - hosts.push_back(host); - }; - - auto process_hosts = [&](fds_xml_ctx_t *hosts_elem) { - const fds_xml_cont *content; - while (fds_xml_next(hosts_elem, &content) != FDS_EOC) { - process_host(content->ptr_ctx); - } - }; - - const fds_xml_cont *content; - while (fds_xml_next(params_elem, &content) != FDS_EOC) { - switch (content->id) { - case MODE: - mode = std::string(content->ptr_string); - break; - case PROTOCOL: - protocol = std::string(content->ptr_string); - break; - case HOSTS: - process_hosts(content->ptr_ctx); - break; - case CONNECTION_BUFFER_SIZE: - connection_buffer_size = content->val_int; - break; - case TEMPLATE_REFRESH_INTERVAL_SECS: - template_refresh_interval_secs = content->val_int; - break; - case TEMPLATE_REFRESH_INTERVAL_BYTES: - template_refresh_interval_bytes = content->val_int; - break; - case RECONNECT_INTERVAL_SECS: - reconnect_interval_secs = content->val_int; - break; - } - } - - /// - /// Check parameters and configure - /// - if (lower(mode) == "all" || lower(mode) == "send to all" || lower(mode) == "send-to-all") { - forwarder.set_forward_mode(ForwardMode::SendToAll); - } else if (lower(mode) == "roundrobin" || lower(mode) == "round robin" || lower(mode) == "round-robin") { - forwarder.set_forward_mode(ForwardMode::RoundRobin); - } else { - throw "Invalid mode '" + mode + "', possible values are: 'roundrobin', 'all'"; - } - - if (lower(protocol) == "udp") { - forwarder.set_transport_protocol(TransProto::Udp); - } else if (lower(protocol) == "tcp") { - forwarder.set_transport_protocol(TransProto::Tcp); - } else { - throw "Invalid protocol '" + protocol + "', possible values are: 'tcp', 'udp'"; - } - - if (connection_buffer_size.has_value()) { - if (connection_buffer_size.value() > 0) { - forwarder.set_connection_buffer_size(connection_buffer_size.value()); - } else { - throw std::string("Invalid connection buffer size"); - } - } else { - forwarder.set_connection_buffer_size(default_connection_buffer_size); - } - - if (template_refresh_interval_secs.has_value()) { - if (template_refresh_interval_secs.value() >= 0) { - if (lower(protocol) == "tcp") { - IPX_CTX_WARNING(log_ctx, "Templates refresh interval is set but transport protocol is TCP"); - } - forwarder.set_template_refresh_interval_secs(template_refresh_interval_secs.value()); - } else { - throw std::string("Invalid template refresh secs interval"); - } - } else { - forwarder.set_template_refresh_interval_secs(default_template_refresh_interval_secs); - } - - if (template_refresh_interval_bytes.has_value()) { - if (template_refresh_interval_bytes.value() >= 0) { - if (lower(protocol) == "tcp") { - IPX_CTX_WARNING(log_ctx, "Templates refresh interval is set but transport protocol is TCP"); - } - forwarder.set_template_refresh_interval_bytes(template_refresh_interval_bytes.value()); - } else { - throw std::string("Invalid template refresh bytes interval"); - } - } else { - forwarder.set_template_refresh_interval_secs(default_template_refresh_interval_bytes); - } - - if (template_refresh_interval_bytes.has_value()) { - if (reconnect_interval_secs.value() >= 0) { - if (lower(protocol) == "udp") { - IPX_CTX_WARNING(log_ctx, "Reconnect interval is set but transport protocol is UDP"); - } - forwarder.set_reconnect_interval(reconnect_interval_secs.value()); - } else { - throw std::string("Invalid reconnect interval"); - } - } else { - forwarder.set_template_refresh_interval_secs(default_reconnect_interval_secs); - } - - for (auto &host : hosts) { - forwarder.add_client(host.address, host.port, host.name); - } -} \ No newline at end of file diff --git a/src/plugins/output/forwarder/src/connector/Connector.cpp b/src/plugins/output/forwarder/src/connector/Connector.cpp new file mode 100644 index 00000000..38f7f2d1 --- /dev/null +++ b/src/plugins/output/forwarder/src/connector/Connector.cpp @@ -0,0 +1,583 @@ +/** + * \file src/plugins/output/forwarder/src/connector/Connector.cpp + * \author Michal Sedlak + * \brief Connector class + * \date 2021 + */ + +/* Copyright (C) 2021 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include "connector/Connector.h" + +#include +#include + +#include +#include +#include + +Connector::Connector(const std::vector &hosts, unsigned int nb_premade_connections, + unsigned int reconnect_secs, ipx_ctx_t *log_ctx) : + m_reconnect_secs(reconnect_secs), + m_log_ctx(log_ctx), + m_nb_premade_connections(nb_premade_connections) +{ + // Set up tasks for premade connections + for (auto &host : hosts) { + for (size_t i = 0; i < nb_premade_connections; i++) { + Task task = {}; + task.params = host; + // The worker thread isn't running yet, so we can access m_tasks from here. + m_tasks.emplace_back(std::move(task)); + } + } + + // Start the worker thread + m_thread = std::thread([this](){ this->run(); }); +} + +Connector::~Connector() +{ + // Let the worker thread know that we're stopping + m_stop_flag = true; + m_statpipe.poke(false); + m_thread.join(); +} + +std::shared_ptr +Connector::get(const ConnectionParams &host) +{ + std::unique_lock lock(m_mutex); + + std::shared_ptr future(std::make_shared()); + m_new_requests.emplace_back(Request{host, future}); + m_statpipe.poke(); + + return future; +} + +/** + * Advance execution of tasks where a poll event occured + */ +void +Connector::process_poll_events() +{ + if (m_pollfds.size() == 0) { + return; + } + + for (size_t i = 0; i < m_pollfds.size() - 1; i++) { // - 1 because the last one is the pipe + if (!m_pollfds[i].revents) { + continue; + } + /* + const char *statestr[] = {"NotStarted", "Connecting", "Connected", "ToBeDeleted"}; + IPX_CTX_INFO(m_log_ctx, "[%d] Poll event %u Task %s:%" PRIu16 " State %s", i, m_pollfds[i].revents, + m_tasks[i].params.address.c_str(), m_tasks[i].params.port, statestr[(int)m_tasks[i].state]); + */ + try { + on_task_poll_event(m_tasks[i], m_pollfds[i].revents); + + } catch (const std::runtime_error &err) { + IPX_CTX_INFO(m_log_ctx, "Connecting to %s:%" PRIu16 " failed - %s", + m_tasks[i].params.address.c_str(), m_tasks[i].params.port, err.what()); + on_task_failed(m_tasks[i]); + } + } +} + +/** + * Start tasks that hasn't been started yet and should be + */ +void +Connector::start_tasks() +{ + time_t now = get_monotonic_time(); + + for (auto &task : m_tasks) { + if (task.state != Task::State::NotStarted || task.start_time > now) { + continue; + } + + try { + on_task_start(task); + + } catch (const std::runtime_error &err) { + IPX_CTX_INFO(m_log_ctx, "Connecting to %s:%" PRIu16 " failed - %s", + task.params.address.c_str(), task.params.port, err.what()); + on_task_failed(task); + } + + } +} + +/** + * Populate the vector of pollfds to listen for events on sockets of tasks that are connecting or connected + */ +void +Connector::setup_pollfds() +{ + m_pollfds.resize(m_tasks.size() + 1); + + size_t i = 0; + for (const auto &task : m_tasks) { + struct pollfd &poll_fd = m_pollfds[i++]; + poll_fd.fd = 0; + poll_fd.events = 0; + poll_fd.revents = 0; + + if (task.state == Task::State::Connecting) { + poll_fd.fd = task.sockfd.get(); + poll_fd.events = POLLOUT; + + } else if (task.state == Task::State::Connected) { + poll_fd.fd = task.sockfd.get(); + } + } + + struct pollfd &poll_fd = m_pollfds[i++]; + poll_fd.fd = m_statpipe.readfd(); + poll_fd.events = POLLIN; +} + +/** + * Process new requests from the caller and delete dead requests + */ +void +Connector::process_requests() +{ + std::unique_lock lock(m_mutex); + + // Delete dead requests, e.g. if the caller requested a connection for a new session, but the session closed before + // the connection was estabilished + m_requests.erase(std::remove_if(m_requests.begin(), m_requests.end(), + [](const Request &request) { return request.future.use_count() == 1; } + ), m_requests.end()); + + // Collect new requests + for (auto &new_request : m_new_requests) { + bool found = false; + + // Check if there is premade connection that can be used to fulfil the request + for (auto &task : m_tasks) { + if (task.state == Task::State::Connected && task.params == new_request.params) { + new_request.future->set(std::move(task.sockfd)); + + task.start_time = 0; + task.state = Task::State::NotStarted; + + found = true; + break; + } + } + + // Create task for the request + if (!found) { + Task task = {}; + task.params = new_request.params; + m_tasks.emplace_back(std::move(task)); + + m_requests.emplace_back(std::move(new_request)); + } + } + + m_new_requests.clear(); +} + +/** + * Delete tasks that are marked for deletion + */ +void +Connector::cleanup_tasks() +{ + m_tasks.erase(std::remove_if(m_tasks.begin(), m_tasks.end(), + [](const Task &task) { return task.state == Task::State::ToBeDeleted; }), + m_tasks.end()); +} + +/** + * Wait for a poll event on any of the task sockets or the status pipe + */ +void +Connector::wait_for_poll_event() +{ + // Wait for one of the sockets or the status fd event + if (poll(m_pollfds.data(), m_pollfds.size(), 1000) < 0) { + char *errbuf; + ipx_strerror(errno, errbuf); + IPX_CTX_ERROR(m_log_ctx, "poll() failed: %s", errbuf); + } + + // Empty the pipe as it's only used to stop the poll + m_statpipe.clear(); +} + +/** + * The main loop + */ +void +Connector::main_loop() +{ + while (!m_stop_flag) { + process_poll_events(); + + process_requests(); + + start_tasks(); + + cleanup_tasks(); + + /* + IPX_CTX_INFO(m_log_ctx, "Tasks: %d, Requests: %d", m_tasks.size(), m_requests.size()); + int i = 0; + for (const auto &task : m_tasks) { + i++; + const char *statestr[] = {"NotStarted", "Connecting", "Connected", "ToBeDeleted"}; + IPX_CTX_INFO(m_log_ctx, "[%d] Task %s:%" PRIu16 " State %s StartTime %d", i, + task.params.address.c_str(), task.params.port, statestr[(int)task.state], task.start_time); + } + */ + setup_pollfds(); + + wait_for_poll_event(); + } +} + +/** + * The entry point of the worker thread + */ +void +Connector::run() +{ + try { + main_loop(); + + } catch (const std::bad_alloc &ex) { + IPX_CTX_ERROR(m_log_ctx, "Caught exception in connector thread: Memory error", 0); + IPX_CTX_ERROR(m_log_ctx, "Fatal error, connector stopped!", 0); + + } catch (const std::runtime_error &ex) { + IPX_CTX_ERROR(m_log_ctx, "Caught exception in connector thread: %s", ex.what()); + IPX_CTX_ERROR(m_log_ctx, "Fatal error, connector stopped!", 0); + + } catch (const std::exception &ex) { + IPX_CTX_ERROR(m_log_ctx, "Caught exception in connector thread: %s", ex.what()); + IPX_CTX_ERROR(m_log_ctx, "Fatal error, connector stopped!", 0); + + } catch (...) { + IPX_CTX_ERROR(m_log_ctx, "Caught exception in connector thread", 0); + IPX_CTX_ERROR(m_log_ctx, "Fatal error, connector stopped!", 0); + } +} + +/**********************************************************************************************************************/ + +/** + * Wrapper around getaddrinfo + */ +static std::unique_ptr +resolve_addrs(const ConnectionParams ¶ms) +{ + struct addrinfo *ai; + struct addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + + switch (params.protocol) { + case Protocol::TCP: + hints.ai_protocol = IPPROTO_TCP; + hints.ai_socktype = SOCK_STREAM; + break; + + case Protocol::UDP: + hints.ai_protocol = IPPROTO_UDP; + hints.ai_socktype = SOCK_DGRAM; + break; + + default: assert(0); + } + + int ret = getaddrinfo(params.address.c_str(), std::to_string(params.port).c_str(), &hints, &ai); + if (ret != 0) { + throw std::runtime_error(std::string("getaddrinfo() failed: ") + gai_strerror(ret)); + } + + return {ai, &freeaddrinfo}; +} + +/** + * Create a socket and begin non-blocking connect to the provided address + */ +static UniqueFd +create_and_connect_socket(struct addrinfo *addr) +{ + UniqueFd sockfd; + int raw_sockfd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol); + if (raw_sockfd < 0) { + throw errno_runtime_error(errno, "socket"); + } + sockfd.reset(raw_sockfd); + + int flags; + if ((flags = fcntl(sockfd.get(), F_GETFL)) == -1) { + throw errno_runtime_error(errno, "fcntl"); + } + if (fcntl(sockfd.get(), F_SETFL, flags | O_NONBLOCK) == -1) { + throw errno_runtime_error(errno, "fcntl"); + } + + if (addr->ai_socktype == SOCK_STREAM) { + int optval = 1; + if (setsockopt(sockfd.get(), SOL_SOCKET, SO_KEEPALIVE, &optval, sizeof(optval)) != 0) { + throw errno_runtime_error(errno, "setsockopt"); + } + } + + if (connect(sockfd.get(), addr->ai_addr, addr->ai_addrlen) != 0 && errno != EINPROGRESS) { + throw errno_runtime_error(errno, "connect"); + } + + return sockfd; +} + +/** + * Check if socket error from a non-blocking operation occured and throw it as a std::runtime_error + */ +static void +throw_if_socket_error(int sockfd) +{ + int optval; + socklen_t optlen = sizeof(optval); + + if (getsockopt(sockfd, SOL_SOCKET, SO_ERROR, &optval, &optlen) == -1) { + throw errno_runtime_error(errno, "getsockopt"); + } + + if (optval != 0) { // optval is the errno of the non-blocking operation + throw errno_runtime_error(errno, "connect"); + } +} + +/** + * Connect to the next address and advance the next address pointer + */ +static UniqueFd +connect_next(struct addrinfo *&next_addr) +{ + assert(next_addr != nullptr); + + while (next_addr) { + struct addrinfo *addr = next_addr; + next_addr = next_addr->ai_next; + try { + // This might fail right away even though it is non-blocking connect + return create_and_connect_socket(addr); + + } catch (const std::runtime_error &err) { + // Ignore unless this is the last address + if (!next_addr) { + throw err; + } + } + } + + return {}; +} + +/** + * Handle task start + */ +void +Connector::on_task_start(Task &task) +{ + assert(task.state == Task::State::NotStarted); + + task.addrs = resolve_addrs(task.params); + task.next_addr = task.addrs.get(); + task.sockfd = connect_next(task.next_addr); + task.state = Task::State::Connecting; +} + +/** + * Handle poll event on the task socket + */ +void +Connector::on_task_poll_event(Task &task, int events) +{ + // If the connection is connected and we got a poll event, the connection probably dropped + if (task.state == Task::State::Connected) { + throw_if_socket_error(task.sockfd.get()); + + if (events & POLLERR) { + // Just in case getsockopt(SO_ERROR) didn't return any error but we got an error from the poll event, + // not sure if this can happen, but just to be safe + throw std::runtime_error("socket error"); + } + + return; + } + + assert(task.state == Task::State::Connecting); + // Otherwise we're connecting and the poll event is either connection success or connection error + try { + throw_if_socket_error(task.sockfd.get()); + // No error -> connection successful + on_task_connected(task); + + } catch (const std::runtime_error &err) { + if (!task.next_addr) { + throw err; + } + task.sockfd = connect_next(task.next_addr); + } + +} + +/** + * Handle successful connection of the task socket + */ +void +Connector::on_task_connected(Task &task) +{ + IPX_CTX_INFO(m_log_ctx, "Connecting to %s:%" PRIu16 " successful", + task.params.address.c_str(), task.params.port); + + std::unique_lock lock(m_mutex); + + // If there is a request for this connection, complete it + for (auto it = m_requests.begin(); it != m_requests.end(); it++) { + Request &request = *it; + if (request.params == task.params && request.future.use_count() > 1) { + request.future->set(std::move(task.sockfd)); + m_requests.erase(it); + + // If this was a premade connection, restart the task + if (!should_restart(task)) { + task.state = Task::State::ToBeDeleted; + } else { + task.start_time = 0; + task.state = Task::State::NotStarted; + } + return; + } + } + + // Count how many extra connections for this host there currently are + unsigned int count = 0; + for (const auto &task_ : m_tasks) { + if (task_.state == Task::State::Connected && task.params == task_.params) { + count++; + } + } + + if (count > m_nb_premade_connections) { + // Too many - discard the connection + task.sockfd.reset(); + task.state = Task::State::ToBeDeleted; + return; + } + + // Keep the connection as extra + task.state = Task::State::Connected; +} + +/** + * Handle failure of the task + */ +void +Connector::on_task_failed(Task &task) +{ + std::unique_lock lock(m_mutex); +#if 0 + // Count how many connections are required to see if we want to restart the task or throw it away. + // This is needed because a connection request may be cancelled before it is resolved, in which case we might + // end up with too many extra connections. + long long required = m_nb_premade_connections; + + // Count requests for this host that are still waiting for a socket + for (const auto &request : m_requests) { + // If the use_count is 1 it's only the Connector holding a reference to it -> the future was cancelled + if (task.params == request.params && request.future.use_count() > 1) { + required++; + } + } + + // How many tasks that are connected or might still successfully connect are there + for (const auto &task_ : m_tasks) { + if (task_.state != Task::State::ToBeDeleted && task.params == task_.params) { + required--; + } + } +#endif + // Don't restart the task, there is no need + if (!should_restart(task)) { + task.state = Task::State::ToBeDeleted; + return; + } + + // Reschedule the failed task to be ran after reconnect interval elapses + time_t now = get_monotonic_time(); + task.start_time = now + m_reconnect_secs; + task.state = Task::State::NotStarted; + + IPX_CTX_INFO(m_log_ctx, "Retrying connection to %s:%" PRIu16 " in %u seconds", + task.params.address.c_str(), task.params.port, m_reconnect_secs); +} + +bool +Connector::should_restart(Task &task) +{ + // Count how many connections are required to see if we want to restart the task or throw it away. + // This is needed because a connection request may be cancelled before it is resolved, in which case we might + // end up with too many extra connections. + long long required = m_nb_premade_connections; + + // Count requests for this host that are still waiting for a socket + for (const auto &request : m_requests) { + // If the use_count is 1 it's only the Connector holding a reference to it -> the future was cancelled + if (task.params == request.params && request.future.use_count() > 1) { + required++; + } + } + + // How many tasks that are connected or might still successfully connect are there + for (const auto &task_ : m_tasks) { + if (&task != &task_ && task_.state != Task::State::ToBeDeleted && task.params == task_.params) { + required--; + } + } + + return required > 0; +} \ No newline at end of file diff --git a/src/plugins/output/forwarder/src/connector/Connector.h b/src/plugins/output/forwarder/src/connector/Connector.h new file mode 100644 index 00000000..1dfe25f4 --- /dev/null +++ b/src/plugins/output/forwarder/src/connector/Connector.h @@ -0,0 +1,180 @@ +/** + * \file src/plugins/output/forwarder/src/connector/Connector.h + * \author Michal Sedlak + * \brief Connector class + * \date 2021 + */ + +/* Copyright (C) 2021 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include "common.h" +#include "connector/Pipe.h" +#include "connector/FutureSocket.h" + +/** + * \brief A connector class that handles socket connections on a separate thread + */ +class Connector { +public: + + /** + * \brief The constructor + * + * \param hosts The list of hosts connections will be made to + * \param nb_premade_connections Number of extra open connections to keep + * \param reconnect_secs The reconnect interval + * \param log_ctx The logging context + */ + Connector(const std::vector &hosts, unsigned int nb_premade_connections, + unsigned int reconnect_secs, ipx_ctx_t *log_ctx); + + // No copying or moving + Connector(const Connector&) = delete; + Connector(Connector&&) = delete; + + /** + * \brief Get a connected socket to the host + * + * \param host The host + * \return A socket that is connected or will be connected in the future + */ + std::shared_ptr + get(const ConnectionParams &host); + + /** + * \brief The destructor + */ + ~Connector(); + +private: + struct Request { + // The connection parameters + ConnectionParams params; + // The future also owned by the caller + std::shared_ptr future; + }; + + struct Task { + enum class State { NotStarted, Connecting, Connected, ToBeDeleted }; + // The connection parameters + ConnectionParams params; + // State of the task + State state = State::NotStarted; + // When should the task start + time_t start_time = 0; + // The socket + UniqueFd sockfd; + // The resolved addresses + std::unique_ptr addrs{nullptr, &freeaddrinfo}; + // The next address to try + struct addrinfo *next_addr = nullptr; + }; + + // Reconnect interval + unsigned int m_reconnect_secs; + // Mutex for shared state + std::mutex m_mutex; + // Requests to be retrieved by the worker thread + std::vector m_new_requests; + // The requests - exclusively handled by the worker thread! + std::vector m_requests; + // The tasks - exclusively handled by the worker thread! + std::vector m_tasks; + // Pipe to notify for status changes + Pipe m_statpipe; + // The worker thread + std::thread m_thread; + // Stop flag for the worker thread + std::atomic m_stop_flag{false}; + // The logging context + ipx_ctx_t *m_log_ctx; + // Number of premade connections to keep + unsigned int m_nb_premade_connections; + // The poll fds + std::vector m_pollfds; + + void + wait_for_poll_event(); + + void + cleanup_tasks(); + + void + process_requests(); + + void + setup_pollfds(); + + void + start_tasks(); + + void + process_poll_events(); + + void + main_loop(); + + void + run(); + + void + on_task_start(Task &task); + + void + on_task_poll_event(Task &task, int events); + + void + on_task_connected(Task &task); + + void + on_task_failed(Task &task); + + bool + should_restart(Task &task); +}; diff --git a/src/plugins/output/forwarder/src/connector/FutureSocket.cpp b/src/plugins/output/forwarder/src/connector/FutureSocket.cpp new file mode 100644 index 00000000..b572cdd5 --- /dev/null +++ b/src/plugins/output/forwarder/src/connector/FutureSocket.cpp @@ -0,0 +1,70 @@ +/** + * \file src/plugins/output/forwarder/src/connector/FutureSocket.cpp + * \author Michal Sedlak + * \date 2021 + */ + +/* Copyright (C) 2021 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include "FutureSocket.h" + +bool +FutureSocket::ready() +{ + std::lock_guard lock(m_mutex); + return m_ready; +} + +UniqueFd +FutureSocket::retrieve() +{ + std::lock_guard lock(m_mutex); + if (!m_ready) { + throw std::runtime_error("result is not ready to be retrieved"); + } + m_ready = false; + return std::move(m_result); +} + +void +FutureSocket::set(UniqueFd result) +{ + std::lock_guard lock(m_mutex); + if (m_ready) { + throw std::runtime_error("result is already set"); + } + m_result = std::move(result); + m_ready = true; +} diff --git a/src/plugins/output/forwarder/src/ConnectionManager.h b/src/plugins/output/forwarder/src/connector/FutureSocket.h similarity index 61% rename from src/plugins/output/forwarder/src/ConnectionManager.h rename to src/plugins/output/forwarder/src/connector/FutureSocket.h index 2fe73237..2ea7f938 100644 --- a/src/plugins/output/forwarder/src/ConnectionManager.h +++ b/src/plugins/output/forwarder/src/connector/FutureSocket.h @@ -1,7 +1,7 @@ /** - * \file src/plugins/output/forwarder/src/ConnectionManager.h + * \file src/plugins/output/forwarder/src/connector/Future.h * \author Michal Sedlak - * \brief Connection manager + * \brief Future socket class * \date 2021 */ @@ -41,58 +41,37 @@ #pragma once -#include "ConnectionParams.h" -#include "Connection.h" -#include "SyncPipe.h" - -#include #include -#include -#include +#include #include -#include -#include - -class Connection; - -static constexpr long DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024; -static constexpr int DEFAULT_RECONNECT_INTERVAL_SECS = 5; -class ConnectionManager -{ -friend class Connection; +#include "common.h" +/** + * \brief Class representing a value that will be set in the future + */ +class FutureSocket { public: - Connection & - add_client(ConnectionParams params); + /** + * \brief Check if the result is ready to be retrieved + */ + bool + ready(); - void - start(); - - void - stop(); + /** + * \brief Retrieve the result + */ + UniqueFd + retrieve(); + /** + * \brief Set the result and make it ready for retrieval + */ void - set_reconnect_interval(int number_of_seconds); + set(UniqueFd result); - void - set_connection_buffer_size(long number_of_bytes); - private: - long connection_buffer_size = DEFAULT_BUFFER_SIZE; - int reconnect_interval_secs = DEFAULT_RECONNECT_INTERVAL_SECS; - std::mutex mutex; - std::vector> active_connections; - std::vector> reconnect_connections; - std::thread send_thread; - std::thread reconnect_thread; - std::condition_variable reconnect_cv; - std::atomic exit_flag { false }; - SyncPipe pipe; - - void - send_loop(); - - void - reconnect_loop(); + UniqueFd m_result; + bool m_ready = false; + std::mutex m_mutex; }; diff --git a/src/plugins/output/forwarder/src/IPFIXMessage.h b/src/plugins/output/forwarder/src/connector/Pipe.cpp similarity index 64% rename from src/plugins/output/forwarder/src/IPFIXMessage.h rename to src/plugins/output/forwarder/src/connector/Pipe.cpp index 3d2fce63..7bf330c9 100644 --- a/src/plugins/output/forwarder/src/IPFIXMessage.h +++ b/src/plugins/output/forwarder/src/connector/Pipe.cpp @@ -1,7 +1,7 @@ /** - * \file src/plugins/output/forwarder/src/IPFIXMessage.h + * \file src/plugins/output/forwarder/src/connector/Pipe.cpp * \author Michal Sedlak - * \brief Simple IPFIX message wrapper + * \brief Pipe class * \date 2021 */ @@ -38,70 +38,50 @@ * if advised of the possibility of such damage. * */ +#include "connector/Pipe.h" -#pragma once +#include +#include -#include - -class IPFIXMessage +Pipe::Pipe() { -public: - IPFIXMessage(ipx_msg_ipfix_t *msg) - : msg(msg) - {} - - const ipx_session * - session() - { - ipx_msg_ctx *msg_ctx = ipx_msg_ipfix_get_ctx(msg); - return msg_ctx->session; + int fd[2]; + if (pipe(fd) != 0) { + throw errno_runtime_error(errno, "pipe"); } - uint8_t * - data() - { - return ipx_msg_ipfix_get_packet(msg); - } + m_readfd.reset(fd[0]); + m_writefd.reset(fd[1]); - fds_ipfix_msg_hdr * - header() - { - return (fds_ipfix_msg_hdr *)data(); - } + int flags; - uint16_t - length() - { - return ntohs(header()->length); + if ((flags = fcntl(m_readfd.get(), F_GETFL)) == -1) { + throw errno_runtime_error(errno, "fcntl"); } - - uint32_t - seq_num() - { - return ntohl(header()->seq_num); + if (fcntl(m_readfd.get(), F_SETFL, flags | O_NONBLOCK) == -1) { + throw errno_runtime_error(errno, "fcntl"); } - int - drec_count() - { - return ipx_msg_ipfix_get_drec_cnt(msg); + if ((flags = fcntl(m_writefd.get(), F_GETFL)) == -1) { + throw errno_runtime_error(errno, "fcntl"); } - - uint32_t - odid() - { - return ntohl(header()->odid); + if (fcntl(m_writefd.get(), F_SETFL, flags | O_NONBLOCK) == -1) { + throw errno_runtime_error(errno, "fcntl"); } +} - const fds_tsnapshot_t * - get_templates_snapshot() - { - if (drec_count() == 0) { - return NULL; - } - return ipx_msg_ipfix_get_drec(msg, 0)->rec.snap; +void +Pipe::poke(bool ignore_error) +{ + ssize_t ret = write(m_writefd.get(), "\x00", 1); + if (ret < 0 && !ignore_error) { + throw errno_runtime_error(errno, "write"); } +} -private: - ipx_msg_ipfix_t *msg; -}; \ No newline at end of file +void +Pipe::clear() +{ + char throwaway[16]; + while (read(m_readfd.get(), throwaway, 16) > 0) {} +} \ No newline at end of file diff --git a/src/plugins/output/forwarder/src/SyncPipe.h b/src/plugins/output/forwarder/src/connector/Pipe.h similarity index 68% rename from src/plugins/output/forwarder/src/SyncPipe.h rename to src/plugins/output/forwarder/src/connector/Pipe.h index aed128f8..cec0793b 100644 --- a/src/plugins/output/forwarder/src/SyncPipe.h +++ b/src/plugins/output/forwarder/src/connector/Pipe.h @@ -1,7 +1,7 @@ /** - * \file src/plugins/output/forwarder/src/SyncPipe.h + * \file src/plugins/output/forwarder/src/connector/Pipe.h * \author Michal Sedlak - * \brief Pipe used for synchronization of threads (when waiting on select) + * \brief Pipe class * \date 2021 */ @@ -41,52 +41,41 @@ #pragma once -#include -#include +#include "common.h" -#include - -class SyncPipe -{ +/** + * \brief Simple utility class around pipe used for interrupting poll + */ +class Pipe { public: - SyncPipe() - { - int pipefd[2]; - if (pipe(pipefd) != 0) { - throw std::string("Cannot create pipe"); - } - readfd = pipefd[0]; - writefd = pipefd[1]; + /** + * \brief The constructor + */ + Pipe(); - int flags = fcntl(readfd, F_GETFL, 0); - fcntl(readfd, F_SETFL, flags | O_NONBLOCK); - } + Pipe(const Pipe &) = delete; + Pipe(Pipe &&) = delete; + /** + * \brief Write to the pipe to trigger its write event + * \param ignore_error Do not throw on write error + * \throw std::runtime_erorr on write error if ignore_error is false + */ void - notify() - { - write(writefd, "A", 1); - } + poke(bool ignore_error = false); + /** + * \brief Read everything from the pipe and throw it away + */ void - clear() - { - char discard_buffer[128]; - while (1) { - int n = read(readfd, discard_buffer, 128); - if (n < 128) { - break; - } - } - } + clear(); - int - get_readfd() - { - return readfd; - } + /** + * \brief Get the read file descriptor + */ + int readfd() const { return m_readfd.get(); } private: - int readfd; - int writefd; + UniqueFd m_readfd; + UniqueFd m_writefd; }; diff --git a/src/plugins/output/forwarder/src/main.cpp b/src/plugins/output/forwarder/src/main.cpp index 9d08162c..1aa4a74c 100644 --- a/src/plugins/output/forwarder/src/main.cpp +++ b/src/plugins/output/forwarder/src/main.cpp @@ -40,7 +40,7 @@ */ #include "Forwarder.h" -#include "config.h" +#include "Config.h" #include @@ -60,14 +60,32 @@ IPX_API struct ipx_plugin_info ipx_plugin_info = { int ipx_plugin_init(ipx_ctx_t *ctx, const char *xml_config) { - auto forwarder = std::unique_ptr(new Forwarder(ctx)); + std::unique_ptr forwarder; + try { - parse_and_configure(ctx, xml_config, *forwarder); - } catch (std::string &error_message) { - IPX_CTX_ERROR(ctx, "%s", error_message.c_str()); - return IPX_ERR_FORMAT; + Config config(xml_config); + forwarder.reset(new Forwarder(config, ctx)); + + } catch (const std::invalid_argument &ex) { + IPX_CTX_ERROR(ctx, "%s", ex.what()); + return IPX_ERR_DENIED; + + } catch (const std::bad_alloc &ex) { + IPX_CTX_ERROR(ctx, "Memory error", 0); + return IPX_ERR_DENIED; + + } catch (const std::runtime_error &ex) { + IPX_CTX_ERROR(ctx, "%s", ex.what()); + return IPX_ERR_DENIED; + + } catch (const std::exception &ex) { + IPX_CTX_ERROR(ctx, "Caught exception %s", ex.what()); + return IPX_ERR_DENIED; + + } catch (...) { + IPX_CTX_ERROR(ctx, "Caught unknown exception", 0); + return IPX_ERR_DENIED; } - forwarder->start(); ipx_msg_mask_t mask = IPX_MSG_IPFIX | IPX_MSG_SESSION; ipx_ctx_subscribe(ctx, &mask, NULL); @@ -79,32 +97,48 @@ ipx_plugin_init(ipx_ctx_t *ctx, const char *xml_config) void ipx_plugin_destroy(ipx_ctx_t *ctx, void *priv) { - (void)ctx; - Forwarder *forwarder = (Forwarder *)(priv); - forwarder->stop(); + (void) ctx; + + Forwarder *forwarder = (Forwarder *) priv; delete forwarder; } int ipx_plugin_process(ipx_ctx_t *ctx, void *priv, ipx_msg_t *msg) { - Forwarder *forwarder = (Forwarder *)(priv); + Forwarder &forwarder = *(Forwarder *) priv; + try { + switch (ipx_msg_get_type(msg)) { + case IPX_MSG_IPFIX: - forwarder->on_ipfix_message(ipx_msg_base2ipfix(msg)); + forwarder.handle_ipfix_message(ipx_msg_base2ipfix(msg)); break; + case IPX_MSG_SESSION: - forwarder->on_session_message(ipx_msg_base2session(msg)); + forwarder.handle_session_message(ipx_msg_base2session(msg)); break; + default: assert(0); } - } catch (std::string &error_message) { - IPX_CTX_ERROR(ctx, "%s", error_message.c_str()); + + } catch (const std::bad_alloc &ex) { + IPX_CTX_ERROR(ctx, "Memory error", 0); + return IPX_ERR_DENIED; + + } catch (const std::runtime_error &ex) { + IPX_CTX_ERROR(ctx, "%s", ex.what()); + return IPX_ERR_DENIED; + + } catch (const std::exception &ex) { + IPX_CTX_ERROR(ctx, "Caught exception %s", ex.what()); + return IPX_ERR_DENIED; + + } catch (...) { + IPX_CTX_ERROR(ctx, "Caught unknown exception", 0); return IPX_ERR_DENIED; - } catch (std::bad_alloc &ex) { - IPX_CTX_ERROR(ctx, "Memory error"); - return IPX_ERR_NOMEM; } + return IPX_OK; } From c9dcadf2052a49055b20d0f6db00e6e01a2683ae Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 8 Feb 2022 18:12:50 +0100 Subject: [PATCH 22/31] Forwarder output plugin: remove preview label --- src/plugins/output/forwarder/README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/output/forwarder/README.rst b/src/plugins/output/forwarder/README.rst index aaeef7a6..427b0970 100644 --- a/src/plugins/output/forwarder/README.rst +++ b/src/plugins/output/forwarder/README.rst @@ -1,5 +1,5 @@ -Forwarder (output plugin) [Preview] -=================================== +Forwarder (output plugin) +========================= This plugin allows forwarding incoming flow records in IPFIX form to other collector in various modes. From c5fecb1e5e5f7b85c438f428bb4381e29130a41f Mon Sep 17 00:00:00 2001 From: Tomas Cejka Date: Fri, 11 Feb 2022 00:26:12 +0100 Subject: [PATCH 23/31] Unirec output: updated and checked list of known UniRec fields --- .../output/unirec/config/unirec-elements.txt | 96 +++++++++++++++---- 1 file changed, 76 insertions(+), 20 deletions(-) diff --git a/extra_plugins/output/unirec/config/unirec-elements.txt b/extra_plugins/output/unirec/config/unirec-elements.txt index 949fa623..7afb6896 100644 --- a/extra_plugins/output/unirec/config/unirec-elements.txt +++ b/extra_plugins/output/unirec/config/unirec-elements.txt @@ -30,11 +30,13 @@ TCP_FLAGS uint8 e0id6 BYTES uint64 e0id1 # Number of bytes in flow PACKETS uint32 e0id2 # Number of packets in flow TTL uint8 e0id192 # IP time to live +TTL_REV uint8 e29305id192 # IP time to live rev TOS uint8 e0id5 # IP type of service TIME_FIRST time e0id150,e0id152,e0id154,e0id156 # Time of the first packet of a flow TIME_LAST time e0id151,e0id153,e0id155,e0id157 # Time of the last packet of a flow DIR_BIT_FIELD uint8 _internal_dbf_ # Bit field used for determining incoming/outgoing flow (1 => Incoming, 0 => Outgoing) LINK_BIT_FIELD uint64 _internal_lbf_ # Bit field of links on which was flow seen + SRC_MAC macaddr e0id56 DST_MAC macaddr e0id80 @@ -43,15 +45,19 @@ BYTES_REV uint64 e29305id1 PACKETS_REV uint32 e29305id2 TCP_FLAGS_REV uint8 e29305id6 +# --- Additional TCP fields --- +TCP_SYN_SIZE uint8 flowmon:tcpSynSize +TCP_SYN_TTL uint8 flowmon:tcpSynTtl + # --- DNS specific fields --- DNS_ANSWERS uint16 cesnet:DNSAnswers # DNS answers DNS_RCODE uint8 cesnet:DNSRCode # DNS rcode -DNS_NAME string cesnet:DNSName # DNS name -DNS_QTYPE uint16 cesnet:DNSQType # DNS qtype -DNS_CLASS uint16 cesnet:DNSClass # DNS class +DNS_Q_NAME string cesnet:DNSName # DNS name +DNS_Q_TYPE uint16 cesnet:DNSQType # DNS qtype +DNS_Q_CLASS uint16 cesnet:DNSClass # DNS class DNS_RR_TTL uint32 cesnet:DNSRRTTL # DNS rr ttl -DNS_RLENGTH uint16 cesnet:DNSRDataLength # DNS rlenght -DNS_RDATA bytes cesnet:DNSRData # DNS rdata +DNS_RR_RLENGTH uint16 cesnet:DNSRDataLength # DNS rlenght +DNS_RR_RDATA bytes cesnet:DNSRData # DNS rdata DNS_PSIZE uint16 cesnet:DNSPSize # DNS payload size DNS_DO uint8 cesnet:DNSRDO # DNS DNSSEC OK bit DNS_ID uint16 cesnet:DNSTransactionID # DNS transaction id @@ -118,6 +124,8 @@ FME_SIP_CALLED_PARTY string flowmon:sipCalledParty FME_SIP_VIA string flowmon:sipVia # SIP VIA # --- HTTP elements --- +HTTP_REQUEST_METHOD_ID string cesnet:httpMethod # HTTP request method + FME_HTTP_UA_OS uint16 flowmon:httpUaOs FME_HTTP_UA_OS_MAJ uint16 flowmon:httpUaOsMaj FME_HTTP_UA_OS_MIN uint16 flowmon:httpUaOsMin @@ -126,15 +134,14 @@ FME_HTTP_UA_APP uint16 flowmon:httpUaApp FME_HTTP_UA_APP_MAJ uint16 flowmon:httpUaAppMaj FME_HTTP_UA_APP_MIN uint16 flowmon:httpUaAppMin FME_HTTP_UA_APP_BLD uint16 flowmon:httpUaAppBld +HTTP_REQUEST_REFERER string flowmon:httpReferer +HTTP_RESPONSE_CONTENT_TYPE string flowmon:httpContentType FME_HTTP_METHOD_MASK uint16 flowmon:httpMethodMask - -HTTP_REQUEST_AGENT string flowmon:httpUserAgent HTTP_REQUEST_HOST string flowmon:httpHost # HTTP(S) request host -HTTP_REQUEST_REFERER string flowmon:httpReferer HTTP_REQUEST_URL string flowmon:httpUrl # HTTP request url -HTTP_REQUEST_METHOD string cesnet:httpMethod # HTTP method text representation -HTTP_RESPONSE_STATUS_CODE uint16 flowmon:httpStatusCode # HTTP response status code -HTTP_RESPONSE_CONTENT_TYPE string flowmon:httpContentType # HTTP ContentType text representation +HTTP_RESPONSE_STATUS_CODE uint32 flowmon:httpStatusCode # HTTP response status code +HTTP_REQUEST_AGENT string flowmon:httpUserAgent + # --- Other fields --- IPV6_TUN_TYPE uint8 e16982id405 # IPv6 tunnel type @@ -168,10 +175,11 @@ FME_TLS_VALIDITY_NOTAFTER int64 flowmon:tlsValidityNotAfter FME_TLS_SIGNATURE_ALG uint16 flowmon:tlsSignatureAlg # tlsSignatureAlg FME_TLS_PUBLIC_KEYALG uint16 flowmon:tlsPublicKeyAlg # tlsPublicKeyAlg FME_TLS_PUBLIC_KEYLENGTH int32 flowmon:tlsPublicKeyLength # tlsPublicKeyLength -FME_TLS_JA_3FINGERPRINT bytes flowmon:tlsJa3Fingerprint # tlsJa3Fingerprint TLS_SNI string cesnet:TLSSNI # Server Name Indication https://en.wikipedia.org/wiki/Server_Name_Indication -TLS_JA_3FINGERPRINT bytes cesnet:tlsJa3Fingerprint # tlsJa3Fingerprint +TLS_JA3_FINGERPRINT bytes cesnet:tlsJa3Fingerprint,flowmon:tlsJa3Fingerprint # tlsJa3Fingerprint + +QUIC_SNI string cesnet:quicSNI # Server Name Indication from QUIC # --- Per-Packet Information elements --- PPI_PKT_LENGTHS uint16* e0id291/cesnet:packetLength # basicList of packet lengths @@ -180,15 +188,15 @@ PPI_PKT_FLAGS uint8* e0id291/cesnet:packetFlag PPI_PKT_DIRECTIONS int8* e0id291/cesnet:packetDirection # basicList of packet directions # --- SSDP Information elements --- -SSDP_LOCATION_PORT uint16 cesnet:SSDPLocationPort -SSDP_SERVER string cesnet:SSDPServer -SSDP_USER_AGENT string cesnet:SSDPUserAgent -SSDP_NT string cesnet:SSDPNT -SSDP_ST string cesnet:SSDPST +SSDP_LOCATION_PORT uint16 cesnet:SSDPLocationPort,flowmon:SSDPLocationPort +SSDP_SERVER string cesnet:SSDPServer,flowmon:SSDPServer +SSDP_USER_AGENT string cesnet:SSDPUserAgent,flowmon:SSDPUserAgent +SSDP_NT string cesnet:SSDPNT,flowmon:SSDPNT +SSDP_ST string cesnet:SSDPST,flowmon:SSDPST # --- DNSDD Information elements --- -DNSSD_QUERIES string cesnet:DNSSDQueries -DNSSD_RESPONSES string cesnet:DNSSDResponses +DNSSD_QUERIES string cesnet:DNSSDQueries,flowmon:DNSSDQuery +DNSSD_RESPONSES string cesnet:DNSSDResponses,flowmon:DNSSDResponse # --- OVPN Information elements --- OVPN_CONF_LEVEL uint8 cesnet:OVPNConfLevel @@ -224,3 +232,51 @@ NB_SUFFIX uint8 cesnet:NBSuffix # --- IDPContent Information elements --- IDP_CONTENT bytes cesnet:IDPContent IDP_CONTENT_REV bytes cesnet:IDPContentRev + +# --- Hists --- +S_PHISTS_SIZES uint32* e0id291/cesnet:phistSrcSizes +S_PHISTS_IPT uint32* e0id291/cesnet:phistSrcInterPacketTime +D_PHISTS_SIZES uint32* e0id291/cesnet:phistDstSizes +D_PHISTS_IPT uint32* e0id291/cesnet:phistDstInterPacketTime + +# --- Bursts --- + +SBI_BRST_BYTES uint32* e0id291/cesnet:burstSrcBytes +SBI_BRST_PACKETS uint32* e0id291/cesnet:burstSrcPackets +SBI_BRST_TIME_START time* e0id291/cesnet:burstSrcTimeStart +SBI_BRST_TIME_STOP time* e0id291/cesnet:burstSrcTimeStop +DBI_BRST_PACKETS uint32* e0id291/cesnet:burstDstPackets +DBI_BRST_BYTES uint32* e0id291/cesnet:burstDstBytes +DBI_BRST_TIME_START time* e0id291/cesnet:burstDstTimeStart +DBI_BRST_TIME_STOP time* e0id291/cesnet:burstDstTimeStop + +# --- BasicPlus --- +L4_TCP_MSS uint32 cesnet:tcpMss +L4_TCP_MSS_REV uint32 cesnet:tcpMssRev +L4_TCP_SYN_SIZE uint16 cesnet:tcpSynSize + +L4_TCP_WIN uint16 e0id186 +L4_TCP_WIN_REV uint16 e29305id186 +L4_TCP_OPTIONS uint64 e0id209 +L4_TCP_OPTIONS_REV uint64 e29305id209 +L3_FLAGS uint8 e0id197 +L3_FLAGS_REV uint8 e29305id197 + +# --- wireguard --- +WG_CONF_LEVEL uint8 cesnet:WGConfLevel +WG_SRC_PEER uint32 cesnet:WGSrcPeer +WG_DST_PEER uint32 cesnet:WGDstPeer + +# --- OSQuery --- +OSQUERY_PROGRAM_NAME string cesnet:OSQueryProgramName +OSQUERY_USERNAME string cesnet:OSQueryUsername +OSQUERY_OS_NAME string cesnet:OSQueryOSName +OSQUERY_OS_MAJOR uint16 cesnet:OSQueryOSMajor +OSQUERY_OS_MINOR uint16 cesnet:OSQueryOSMinor +OSQUERY_OS_BUILD string cesnet:OSQueryOSBuild +OSQUERY_OS_PLATFORM string cesnet:OSQueryOSPlatform +OSQUERY_OS_PLATFORM_LIKE string cesnet:OSQueryOSPlatformLike +OSQUERY_OS_ARCH string cesnet:OSQueryOSArch +OSQUERY_KERNEL_VERSION string cesnet:OSQueryKernelVersion +OSQUERY_SYSTEM_HOSTNAME string cesnet:OSQuerySystemHostname + From ef837bf7fe146fb8568deeaae2afed6d8f48e21e Mon Sep 17 00:00:00 2001 From: Tomas Cejka Date: Wed, 23 Feb 2022 18:27:36 +0100 Subject: [PATCH 24/31] Unirec output: add missing FLOW_END_REASON field --- extra_plugins/output/unirec/config/unirec-elements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extra_plugins/output/unirec/config/unirec-elements.txt b/extra_plugins/output/unirec/config/unirec-elements.txt index 7afb6896..cbe5eab9 100644 --- a/extra_plugins/output/unirec/config/unirec-elements.txt +++ b/extra_plugins/output/unirec/config/unirec-elements.txt @@ -40,6 +40,8 @@ LINK_BIT_FIELD uint64 _internal_lbf_ SRC_MAC macaddr e0id56 DST_MAC macaddr e0id80 +FLOW_END_REASON uint8 iana:flowEndReason # Reason of exporting the flow record, see https://www.iana.org/assignments/ipfix/ipfix.xhtml#ipfix-flow-end-reason + # --- Additional biflow fields --- BYTES_REV uint64 e29305id1 PACKETS_REV uint32 e29305id2 From 686645574aa662e2247970050fa62860cac839ce Mon Sep 17 00:00:00 2001 From: Tomas Cejka Date: Wed, 23 Feb 2022 18:28:56 +0100 Subject: [PATCH 25/31] Unirec output: removed duplicit TLS JA3 element IPFIX ipfixprobe uses flowmon PEN for TLS JA3, which has the same format and semantic. --- extra_plugins/output/unirec/config/unirec-elements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra_plugins/output/unirec/config/unirec-elements.txt b/extra_plugins/output/unirec/config/unirec-elements.txt index cbe5eab9..9dd22bba 100644 --- a/extra_plugins/output/unirec/config/unirec-elements.txt +++ b/extra_plugins/output/unirec/config/unirec-elements.txt @@ -179,7 +179,7 @@ FME_TLS_PUBLIC_KEYALG uint16 flowmon:tlsPublicKeyAlg FME_TLS_PUBLIC_KEYLENGTH int32 flowmon:tlsPublicKeyLength # tlsPublicKeyLength TLS_SNI string cesnet:TLSSNI # Server Name Indication https://en.wikipedia.org/wiki/Server_Name_Indication -TLS_JA3_FINGERPRINT bytes cesnet:tlsJa3Fingerprint,flowmon:tlsJa3Fingerprint # tlsJa3Fingerprint +TLS_JA3_FINGERPRINT bytes flowmon:tlsJa3Fingerprint # tlsJa3Fingerprint QUIC_SNI string cesnet:quicSNI # Server Name Indication from QUIC From 6a5f768464dfae5847204ef9fdc314eba0fbd88b Mon Sep 17 00:00:00 2001 From: Tomas Cejka Date: Wed, 23 Feb 2022 18:34:34 +0100 Subject: [PATCH 26/31] Unirec output: add QUIC elements for User-Agent and Version --- extra_plugins/output/unirec/config/unirec-elements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extra_plugins/output/unirec/config/unirec-elements.txt b/extra_plugins/output/unirec/config/unirec-elements.txt index 9dd22bba..f6f97dcd 100644 --- a/extra_plugins/output/unirec/config/unirec-elements.txt +++ b/extra_plugins/output/unirec/config/unirec-elements.txt @@ -182,6 +182,8 @@ TLS_SNI string cesnet:TLSSNI TLS_JA3_FINGERPRINT bytes flowmon:tlsJa3Fingerprint # tlsJa3Fingerprint QUIC_SNI string cesnet:quicSNI # Server Name Indication from QUIC +QUIC_USER_AGENT string cesnet:quicUserAgent # User-Agent value extracted from decrypted QUIC header +QUIC_VERSION uint32 cesnet:quicVersion # Version of QUIC protocol extracted from decrypted QUIC header # --- Per-Packet Information elements --- PPI_PKT_LENGTHS uint16* e0id291/cesnet:packetLength # basicList of packet lengths From 9d871c2671196debcabb3c57aac4ac9fc0ab1d98 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 3 Mar 2022 21:45:48 +0100 Subject: [PATCH 27/31] FDS output: add .tmp suffix for files in progress --- src/plugins/output/fds/src/Storage.cpp | 13 ++++++++++++- src/plugins/output/fds/src/Storage.hpp | 4 +++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/plugins/output/fds/src/Storage.cpp b/src/plugins/output/fds/src/Storage.cpp index 30db3d25..ef55e095 100644 --- a/src/plugins/output/fds/src/Storage.cpp +++ b/src/plugins/output/fds/src/Storage.cpp @@ -46,6 +46,11 @@ Storage::Storage(ipx_ctx_t *ctx, const Config &cfg) : m_ctx(ctx), m_path(cfg.m_p m_flags |= FDS_FILE_APPEND; } +Storage::~Storage() +{ + window_close(); +} + void Storage::window_new(time_t ts) { @@ -53,7 +58,8 @@ Storage::window_new(time_t ts) window_close(); // Open new file - const std::string new_file = filename_gen(ts); + const std::string new_file = filename_gen(ts) + ".tmp"; + m_file_name = new_file; std::unique_ptr new_file_cpy(strdup(new_file.c_str()), &free); char *dir2create; @@ -82,6 +88,11 @@ Storage::window_close() { m_file.reset(); m_session2params.clear(); + if (!m_file_name.empty()) { + std::string new_file_name(m_file_name.begin(), m_file_name.end() - std::string(".tmp").size()); + std::rename(m_file_name.c_str(), new_file_name.c_str()); + m_file_name.clear(); + } } void diff --git a/src/plugins/output/fds/src/Storage.hpp b/src/plugins/output/fds/src/Storage.hpp index 29b0528e..56bbcc14 100644 --- a/src/plugins/output/fds/src/Storage.hpp +++ b/src/plugins/output/fds/src/Storage.hpp @@ -36,7 +36,7 @@ class Storage { * @throw FDS_exception if @p path directory doesn't exist in the system */ Storage(ipx_ctx_t *ctx, const Config &cfg); - virtual ~Storage() = default; + virtual ~Storage(); // Disable copy constructors Storage(const Storage &other) = delete; @@ -105,6 +105,8 @@ class Storage { /// Output FDS file std::unique_ptr m_file = {nullptr, &fds_file_close}; + /// Output FDS file name + std::string m_file_name; /// Mapping of Transport Sessions to FDS specific parameters std::map m_session2params; From ede37d9d12ceebbc83f938b99c9b86c9fb89857a Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 3 Mar 2022 22:07:41 +0100 Subject: [PATCH 28/31] FDS output: proper behavior on file open failure --- src/plugins/output/fds/src/Storage.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/output/fds/src/Storage.cpp b/src/plugins/output/fds/src/Storage.cpp index ef55e095..236c4bb6 100644 --- a/src/plugins/output/fds/src/Storage.cpp +++ b/src/plugins/output/fds/src/Storage.cpp @@ -86,9 +86,10 @@ Storage::window_new(time_t ts) void Storage::window_close() { + bool file_opened = (m_file.get() != nullptr); m_file.reset(); m_session2params.clear(); - if (!m_file_name.empty()) { + if (file_opened) { std::string new_file_name(m_file_name.begin(), m_file_name.end() - std::string(".tmp").size()); std::rename(m_file_name.c_str(), new_file_name.c_str()); m_file_name.clear(); From bda12efa0f5fc2599c9cf88c816683c680636293 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 3 Mar 2022 22:15:30 +0100 Subject: [PATCH 29/31] FDS output: make tmp suffix a constant --- src/plugins/output/fds/src/Storage.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/plugins/output/fds/src/Storage.cpp b/src/plugins/output/fds/src/Storage.cpp index 236c4bb6..5867c344 100644 --- a/src/plugins/output/fds/src/Storage.cpp +++ b/src/plugins/output/fds/src/Storage.cpp @@ -17,6 +17,8 @@ #include #include "Storage.hpp" +const std::string TMP_SUFFIX = ".tmp"; + Storage::Storage(ipx_ctx_t *ctx, const Config &cfg) : m_ctx(ctx), m_path(cfg.m_path) { // Check if the directory exists @@ -58,7 +60,7 @@ Storage::window_new(time_t ts) window_close(); // Open new file - const std::string new_file = filename_gen(ts) + ".tmp"; + const std::string new_file = filename_gen(ts) + TMP_SUFFIX; m_file_name = new_file; std::unique_ptr new_file_cpy(strdup(new_file.c_str()), &free); @@ -90,7 +92,7 @@ Storage::window_close() m_file.reset(); m_session2params.clear(); if (file_opened) { - std::string new_file_name(m_file_name.begin(), m_file_name.end() - std::string(".tmp").size()); + std::string new_file_name(m_file_name.begin(), m_file_name.end() - TMP_SUFFIX.size()); std::rename(m_file_name.c_str(), new_file_name.c_str()); m_file_name.clear(); } From a929299ebc6844f12ccee29bbabbaa30ee64b532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Hut=C3=A1k?= Date: Fri, 4 Mar 2022 16:41:41 +0100 Subject: [PATCH 30/31] Unirec output: introduce mappingFile parameter --- extra_plugins/output/unirec/README.rst | 4 +++ .../output/unirec/src/configuration.c | 23 +++++++++++++++ .../output/unirec/src/configuration.h | 4 ++- .../output/unirec/src/unirecplugin.c | 29 ++++--------------- 4 files changed, 36 insertions(+), 24 deletions(-) diff --git a/extra_plugins/output/unirec/README.rst b/extra_plugins/output/unirec/README.rst index 39d383ae..36cab628 100644 --- a/extra_plugins/output/unirec/README.rst +++ b/extra_plugins/output/unirec/README.rst @@ -123,6 +123,10 @@ Parameters Specification of interface type and its parameters. For more details, see section "Output interface types". +:``mappingFile``: + Path to configuration file with mapping IPFIX fields to UniRec fields. If the parameter is + not defined, the default configuration file is used. See section "UniRec configuration file". + Output interface types ---------------------- Exactly one of the following output type must be defined in the instance configuration of this diff --git a/extra_plugins/output/unirec/src/configuration.c b/extra_plugins/output/unirec/src/configuration.c index bc40455e..41e6930c 100644 --- a/extra_plugins/output/unirec/src/configuration.c +++ b/extra_plugins/output/unirec/src/configuration.c @@ -63,6 +63,8 @@ enum cfg_timeout_mode { CFG_TIMEOUT_HALF_WAIT = -3 /**< Block only if some client is connected */ }; +/** Filename of IPFIX-to-UniRec */ +#define DEF_CONF_FILENAME "unirec-elements.txt" /** Default maximum number of connections over TCP/TCP-TLS/Unix */ #define DEF_MAX_CONNECTIONS 64 /** Default output interface timeout */ @@ -84,6 +86,7 @@ struct ifc_common { /* * + * /etc/ipfixcol2/unirec-elements.txt * DST_IP,SRC_IP,BYTES,DST_PORT,?TCP_FLAGS,SRC_PORT,PROTOCOL * true * @@ -123,6 +126,7 @@ enum params_xml_nodes { // Main parameters NODE_UNIREC_FMT = 1, NODE_BIFLOW_SPLIT, + NODE_MAPPING_FILE, NODE_TRAP_COMMON, NODE_TRAP_SPEC, // TRAP common parameters @@ -208,6 +212,7 @@ static const struct fds_xml_args args_params[] = { FDS_OPTS_ROOT("params"), FDS_OPTS_ELEM(NODE_UNIREC_FMT, "uniRecFormat", FDS_OPTS_T_STRING, 0), FDS_OPTS_ELEM(NODE_BIFLOW_SPLIT, "splitBiflow", FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT), + FDS_OPTS_ELEM(NODE_MAPPING_FILE, "mappingFile", FDS_OPTS_T_STRING, FDS_OPTS_P_OPT), FDS_OPTS_NESTED(NODE_TRAP_COMMON, "trapIfcCommon", args_trap_common, FDS_OPTS_P_OPT), FDS_OPTS_NESTED(NODE_TRAP_SPEC, "trapIfcSpec", args_trap_spec, 0), FDS_OPTS_END @@ -781,6 +786,13 @@ cfg_parse_params(ipx_ctx_t *ctx, fds_xml_ctx_t *root, struct conf_params *cfg) // Set default values cfg->biflow_split = true; + cfg->mapping_file = NULL; + + rc = cfg_str_append(&cfg->mapping_file, "%s/%s", ipx_api_cfg_dir(), DEF_CONF_FILENAME); + if (rc != FDS_OK) { + IPX_CTX_ERROR(ctx, "Unable to allocate memory (%s:%d)", __FILE__, __LINE__); + return rc; + } // Set default TRAP common parameters struct ifc_common common; @@ -807,6 +819,16 @@ cfg_parse_params(ipx_ctx_t *ctx, fds_xml_ctx_t *root, struct conf_params *cfg) assert(content->type == FDS_OPTS_T_BOOL); cfg->biflow_split = content->val_bool; break; + case NODE_MAPPING_FILE: + // Mapping file + assert(content->type == FDS_OPTS_T_STRING); + free(cfg->mapping_file); + cfg->mapping_file = strdup(content->ptr_string); + if (cfg->mapping_file == NULL) { + IPX_CTX_ERROR(ctx, "Unable to allocate memory (%s:%d)", __FILE__, __LINE__); + return IPX_ERR_NOMEM; + } + break; case NODE_TRAP_SPEC: // TRAP output interface specifier assert(content->type == FDS_OPTS_T_CONTEXT); @@ -920,6 +942,7 @@ configuration_free(struct conf_params *cfg) return; } + free(cfg->mapping_file); free(cfg->trap_ifc_spec); free(cfg->unirec_fmt); free(cfg->unirec_spec); diff --git a/extra_plugins/output/unirec/src/configuration.h b/extra_plugins/output/unirec/src/configuration.h index c9a5eb91..8890bbd3 100644 --- a/extra_plugins/output/unirec/src/configuration.h +++ b/extra_plugins/output/unirec/src/configuration.h @@ -46,7 +46,9 @@ * \brief Structure for a configuration parsed from XML */ struct conf_params { - /** Prepared TRAP interface specification string */ + /** Path to IPFIX-to-UniRec mapping file */ + char *mapping_file; + /** Prepared TRAP interface specification string */ char *trap_ifc_spec; /** * TRAP interface UniRec template diff --git a/extra_plugins/output/unirec/src/unirecplugin.c b/extra_plugins/output/unirec/src/unirecplugin.c index dca9f8b3..e336f786 100644 --- a/extra_plugins/output/unirec/src/unirecplugin.c +++ b/extra_plugins/output/unirec/src/unirecplugin.c @@ -48,8 +48,6 @@ #include "configuration.h" #include "map.h" -/** Filename of IPFIX-to-UniRec */ -#define CONF_FILENAME "unirec-elements.txt" /** Name of TRAP context that belongs to the plugin */ #define PLUGIN_TRAP_NAME "IPFIXcol2-UniRec" /** Description of the TRAP context that belongs to the plugin */ @@ -100,41 +98,26 @@ struct conf_unirec { /** * \brief Get the IPFIX-to-UniRec conversion database * \param ctx Plugin context + * \param file Path to a file with IPFIX-to-UniRec mapping * \return Conversion table or NULL (an error has occurred) */ static map_t * -ipfix2unirec_db(ipx_ctx_t *ctx) +ipfix2unirec_db(ipx_ctx_t *ctx, const char *file) { - const char *path = ipx_api_cfg_dir(); - const size_t full_size = strlen(path) + strlen(CONF_FILENAME) + 2; // 2 = '/' + '\0' - char *full_path = malloc(full_size * sizeof(char)); - if (!full_path) { - IPX_CTX_ERROR(ctx, "Unable to allocate memory (%s:%d)", __FILE__, __LINE__); - return NULL; - } - - int ret_val = snprintf(full_path, full_size, "%s/%s", path, CONF_FILENAME); - if (ret_val < 0 || ((size_t) ret_val) >= full_size) { - IPX_CTX_ERROR(ctx, "Failed to generate a configuration path (internal error)", '\0'); - free(full_path); - return NULL; - } - map_t *map = map_init(ipx_ctx_iemgr_get(ctx)); if (!map) { IPX_CTX_ERROR(ctx, "Failed to initialize conversion map! (%s:%d)", __FILE__, __LINE__); - free(full_path); return NULL; } - if (map_load(map, full_path) != IPX_OK) { + IPX_CTX_INFO(ctx, "Loading IPFIX-to-UniRec mapping file '%s'", file); + + if (map_load(map, file) != IPX_OK) { IPX_CTX_ERROR(ctx, "Failed to initialize conversion database: %s", map_last_error(map)); map_destroy(map); - free(full_path); return NULL; } - free(full_path); return map; } @@ -317,7 +300,7 @@ ipx_plugin_init(ipx_ctx_t *ctx, const char *params) conf->params = parsed_params; // Load IPFIX-to-UniRec conversion database - map_t *conv_db = ipfix2unirec_db(ctx); + map_t *conv_db = ipfix2unirec_db(ctx, parsed_params->mapping_file); if (!conv_db) { configuration_free(parsed_params); free(conf); From 83ac59d78fb3cf80ca44fd939ee45f5921b08088 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Fri, 11 Mar 2022 11:27:20 +0100 Subject: [PATCH 31/31] CMake: bump collector version to 2.3.0 --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 830a3795..8dd95de3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,8 +14,8 @@ endif() # Versions and other informations set(IPFIXCOL_VERSION_MAJOR 2) -set(IPFIXCOL_VERSION_MINOR 2) -set(IPFIXCOL_VERSION_PATCH 1) +set(IPFIXCOL_VERSION_MINOR 3) +set(IPFIXCOL_VERSION_PATCH 0) set(IPFIXCOL_VERSION ${IPFIXCOL_VERSION_MAJOR}.${IPFIXCOL_VERSION_MINOR}.${IPFIXCOL_VERSION_PATCH})