From 5e8067c14d02b97a4194b25ddb02411d5c0f202c Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Wed, 16 Apr 2025 12:47:13 +0200 Subject: [PATCH 01/40] Core - destroy leftover periodic messages on plugin destruction This fixes a memory leak and a warning that occured on termination because of unprocessed periodic messages still being left in the feedback pipe. We exhaust the pipe and destroy any leftover messages now. --- src/core/fpipe.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/core/fpipe.c b/src/core/fpipe.c index dc5e60185..a6411a557 100644 --- a/src/core/fpipe.c +++ b/src/core/fpipe.c @@ -93,9 +93,19 @@ ipx_fpipe_create() void ipx_fpipe_destroy(ipx_fpipe_t *fpipe) { - if (fpipe->cnt != 0) { + // Destroy any leftover periodic messages + ipx_msg_t *msg; + int err_cnt = 0; + while ((msg = ipx_fpipe_read(fpipe)) != NULL) { + if (ipx_msg_get_type(msg) != IPX_MSG_PERIODIC) { + err_cnt++; + } + ipx_msg_destroy(msg); + } + + if (err_cnt > 0) { IPX_WARNING(fpipe_str, "Destroying of a pipe that still contains %" PRIu32 " unprocessed " - "message(s)!", fpipe->cnt); + "non-periodic message(s)!", fpipe->cnt); } // Close the pipe and destroy the structure From 52d09ea7e6c6bacde575f3df8145b366c0177423 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 14 Nov 2024 12:24:15 +0100 Subject: [PATCH 02/40] Clickhouse - introduce docs --- README.rst | 2 + extra_plugins/output/clickhouse/README.rst | 315 ++++++++++++++++++ .../doc/ipfixcol2-clickhouse-output.7.rst | 20 ++ 3 files changed, 337 insertions(+) create mode 100644 extra_plugins/output/clickhouse/README.rst create mode 100644 extra_plugins/output/clickhouse/doc/ipfixcol2-clickhouse-output.7.rst diff --git a/README.rst b/README.rst index f7a81af16..b1c14d8d2 100644 --- a/README.rst +++ b/README.rst @@ -64,6 +64,8 @@ network interface and a port. Multiple instances of these plugins can run concur format for long-term preservation - `UniRec `_ (*) - send flow records in UniRec format via TRAP communication interface (into Nemea modules) +- `ClickHouse `_ (*) - insert flow records + into a ClickHouse database \* Must be installed individually due to extra dependencies diff --git a/extra_plugins/output/clickhouse/README.rst b/extra_plugins/output/clickhouse/README.rst new file mode 100644 index 000000000..f8b127af2 --- /dev/null +++ b/extra_plugins/output/clickhouse/README.rst @@ -0,0 +1,315 @@ +clickhouse (output plugin) +=========================== + +The plugin converts and stores IPFIX flow records into a ClickHouse database. +It is designed for high-performance environments, ensuring efficient data +handling and storage. + +Key Features +------------ + +- **High-Speed Export**: The conversion and export of data to the database are + performed at the binary level (i.e., without conversion to text SQL + commands). This enables export speeds of hundreds of thousands or even a + million flow records per second (*performance depends on machine + configuration*). + +- **Customizable Data Mapping**: Users can configure the plugin to send any + IPFIX/NetFlow items to the database by specifying the item name and mapping + it to the corresponding database column. + +- **High Availability (HA) Support**: The plugin supports sending data to + multiple ClickHouse endpoints. In case of a failure at one endpoint, data is + automatically redirected to the next available endpoint. This failover + mechanism ensures reliability in HA deployments. Note: This is not a + round-robin distribution but a failover strategy. + +How to build +------------ + +By default, the plugin is not distributed with IPFIXcol due to extra dependencies. +To build the plugin, IPFIXcol2 (and its header files) must be installed on your system. + +Finally, compile and install the plugin: + +.. code-block:: sh + + $ mkdir build && cd build && cmake .. + $ make + # make install + +Usage +------ + +The plugin expects the ClickHouse database to already contain the table with +appropriate schema corresponding to the configuration entered. The existence +and schema of the table is checked after initiating connection to the database +and an error is displayed if there is a mismatch. The table is not +automatically created. + +To run the example configuration below, you can create the ClickHouse table +using the following SQL query: + +.. code-block:: sql + + CREATE TABLE ipfixcol2.flows ( + odid UInt32, + srcip IPv6, + dstip IPv6, + flowstart DateTime64(9), + flowend DateTime64(9), + sourceTransportPort UInt16, + destinationTransportPort UInt16, + protocolIdentifier UInt8, + octetDeltaCount UInt64, + packetDeltaCount UInt64, + INDEX srcipindex srcip TYPE bloom_filter GRANULARITY 16, + INDEX dstipindex dstip TYPE bloom_filter GRANULARITY 16 + ) + ENGINE = MergeTree + PARTITION BY toStartOfInterval(flowstart, INTERVAL 1 HOUR) + ORDER BY flowstart + +The following ClickHouse column types are expected to store the following IPFIX element types: + +.. list-table:: Mapping of IPFIX types to ClickHouse column types + + * - **IPFIX Abstract Data Type** + - **ClickHouse Column Type** + * - unsigned8 + - UInt8 + * - unsigned16 + - UInt16 + * - unsigned32 + - UInt32 + * - unsigned64 + - UInt64 + * - signed8 + - Int8 + * - signed16 + - Int16 + * - signed32 + - Int32 + * - signed64 + - Int64 + * - ipv4Address + - IPv4 + * - ipv6Address + - IPv6 + * - macAddress + - UInt64 + * - dateTimeNanoseconds + - DateTime64(9) + * - dateTimeMicroseconds + - DateTime64(6) + * - dateTimeMilliseconds + - DateTime64(3) + * - dateTimeSeconds + - DateTime + * - string + - String + +In case the field is an alias mapping to multiple IPFIX elements of compatible +types, the resulting type is unified to the type with higher precision, i.e. +the type that can hold both of the values without data loss. To unify IPv4 and +IPv6 addresses as one type, the IPv4 is stored as an IPv6 value as a IPv4 +mapped IPv6 address. + +To be able to retrieve the stored values in a human-friendly format, the +following ClickHouse functions can be defined and used: + +.. code-block:: sql + + CREATE FUNCTION ipToString AS (ip) -> + if(isIPAddressInRange(toString(ip), '::ffff:0.0.0.0/96'), toString(toIPv4(ip)), toString(ip)); + + CREATE FUNCTION macToString AS (mac) -> + concat( + lpad(hex(bitAnd(bitShiftRight(mac, 40), 0xFF)), 2, '0'), ':', + lpad(hex(bitAnd(bitShiftRight(mac, 32), 0xFF)), 2, '0'), ':', + lpad(hex(bitAnd(bitShiftRight(mac, 24), 0xFF)), 2, '0'), ':', + lpad(hex(bitAnd(bitShiftRight(mac, 16), 0xFF)), 2, '0'), ':', + lpad(hex(bitAnd(bitShiftRight(mac, 8), 0xFF)), 2, '0'), ':', + lpad(hex(bitAnd(mac, 0xFF)), 2, '0') + ); + + +Example configuration +--------------------- + +.. code-block:: xml + + + ClickHouse output + clickhouse + + + + + + clickhouse.example.com + 9000 + + + ipfixcol2 + ipfixcol2 + ipfixcol2 + flows
+
+ 8 + 64 + 100000 + true + true + + + + odid + + + + srcip + + + dstip + + + flowstart + + + flowend + + + + srcport + sourceTransportPort + + + dstport + destinationTransportPort + + + + protocolIdentifier + + + octetDeltaCount + + + packetDeltaCount + + +
+
+ +**Warning**: The database and the table with the appropriate schema must already exist. +It will not be created automatically. + +Parameters +---------- + +:``connection``: + The database connection parameters. + + :``endpoints``: + The possible endpoints data can be sent to, i.e. all the replicas of a + particular shard. In case one endpoint is unreachable, another one is used. + + :``endpoint``: + Connection parameters of one endpoint. + + :``host``: + The ClickHouse database host as a domain name or an IP address. + + :``port``: + The port of the ClickHouse database. [default: 9000] + + :``username``:" + The database username. + + :``password``: + The database password. + + :``database``: + The database name where the specified table is present. + + :``table``: + The name of the table to insert the data into. + +:``splitBiflow``: + When true, biflow records are split into two uniflow records. [default: true] + +:``biflowEmptyAutoignore``: + When true and ``splitBiflow`` is active, the uniflow records resulting from + the split are also checked for emptiness and are omitted if empty. A flow + is considered empty when ``octetDeltaCount = 0`` or ``packetDeltaCount = 0``. + This exists because some IPFIX probes may export uniflow records as biflow + with the reverse direction always empty, resulting in a large amount of + empty flow records. + [default: true] + +:``blocks``: + Number of data blocks in circulation. Each block is de-facto a memory + buffer that the rows are written to before being sent out to the ClickHouse + database. [default: 64] + +:``inserterThreads``: + Number of threads used for data insertion to ClickHouse. In other words, + the number of ClickHouse connections that are concurrently used. [default: 8] + +:``blockInsertThreshold``: + Number of rows to be buffered into a block before the block is sent out to + be inserted into the database. [default: 100000] + +:``blockInsertMaxDelaySecs``: + Maximum number of seconds to wait before a block gets sent out to be + inserted into the database even if the threshold has not been reached yet. + [default: 10] + +:``nonblocking``: + This option dictates what happens when all the blocks (buffers) are full. + If true, the processing thread is not blocked, and some data is dropped to + maintain flow of data. + If false, the processing thread is blocked, waiting until a block becomes + available. [default: true] + +:``columns``: + The fields that each row will consist of. + + :``column``: + + :``name``: + Name of the column in the database. Also the source field if source + is not explicitly defined. + + :``nullable``: + Whether null should be a special value. If false, zero value of the + corresponding data type is used as null. Turning this option on + might negatively affect performance. [default: false] + + :``source``: + An IPFIX element name or an alias. If not present, name is used. + Aliases and IPFIX elements can be found + `here `_. + List of standard IPFIX element names can be also found + `here `_. + [default: same as name] + +Performance tuning +------------------ + +In case you are having performance issues with the default values, try +increasing `blockInsertThreshold`, `blocks` and `inserterThreads` configuration +parameters. + +For example based on our testing, the following values should result in better +performance at the cost of higher memory usage: + +.. code-block:: xml + + 16 + 128 + 500000 + +You can further experiment with the values based on your input characteristics +and your machine specifications. diff --git a/extra_plugins/output/clickhouse/doc/ipfixcol2-clickhouse-output.7.rst b/extra_plugins/output/clickhouse/doc/ipfixcol2-clickhouse-output.7.rst new file mode 100644 index 000000000..9b35fa5ad --- /dev/null +++ b/extra_plugins/output/clickhouse/doc/ipfixcol2-clickhouse-output.7.rst @@ -0,0 +1,20 @@ +============================= + ipfixcol2-clickhouse-output +============================= + +-------------------------- +ClickHouse (output plugin) +-------------------------- + +:Author: Michal Sedlak (sedlakm@cesnet.cz) +:Date: 2024-11-04 +:Copyright: Copyright © 2024 CESNET, z.s.p.o. +:Version: 1.0 +:Manual section: 7 +:Manual group: IPFIXcol collector + +Description +----------- + +.. include:: ../README.rst + :start-line: 3 From 46fb9338a99cd94978f4f2015c22ef986f28d29d Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 14 Nov 2024 12:24:37 +0100 Subject: [PATCH 03/40] Clickhouse - introduce config --- .../output/clickhouse/src/config.cpp | 271 ++++++++++++++++++ extra_plugins/output/clickhouse/src/config.h | 67 +++++ 2 files changed, 338 insertions(+) create mode 100644 extra_plugins/output/clickhouse/src/config.cpp create mode 100644 extra_plugins/output/clickhouse/src/config.h diff --git a/extra_plugins/output/clickhouse/src/config.cpp b/extra_plugins/output/clickhouse/src/config.cpp new file mode 100644 index 000000000..a636f6c81 --- /dev/null +++ b/extra_plugins/output/clickhouse/src/config.cpp @@ -0,0 +1,271 @@ +/** + * @file + * @author Michal Sedlak + * @brief Configuration parsing and representation + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "config.h" + +#include + +#include +#include +#include +#include + +namespace args { + +enum { + CONNECTION, + ENDPOINTS, + ENDPOINT, + HOST, + PORT, + USER, + PASSWORD, + DATABASE, + TABLE, + COLUMNS, + COLUMN, + NAME, + SOURCE, + NULLABLE, + INSERTER_THREADS, + BLOCKS, + BLOCK_INSERT_THRESHOLD, + BLOCK_INSERT_MAX_DELAY_SECS, + SPLIT_BIFLOW, + BIFLOW_EMPTY_AUTOIGNORE, + NONBLOCKING, +}; + + +static const struct fds_xml_args column[] = { + FDS_OPTS_ELEM(NAME, "name", FDS_OPTS_T_STRING, 0), + FDS_OPTS_ELEM(SOURCE, "source", FDS_OPTS_T_STRING, FDS_OPTS_P_OPT), + FDS_OPTS_ELEM(NULLABLE, "nullable", FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT), + FDS_OPTS_END, +}; + +static const struct fds_xml_args columns[] = { + FDS_OPTS_NESTED(COLUMN, "column", column, FDS_OPTS_P_MULTI), + FDS_OPTS_END, +}; + +static const struct fds_xml_args endpoint[] = { + FDS_OPTS_ELEM (HOST, "host", FDS_OPTS_T_STRING, 0), + FDS_OPTS_ELEM (PORT, "port", FDS_OPTS_T_UINT, FDS_OPTS_P_OPT), + FDS_OPTS_END, +}; + +static const struct fds_xml_args endpoints[] = { + FDS_OPTS_NESTED(ENDPOINT, "endpoint", endpoint, FDS_OPTS_P_MULTI), + FDS_OPTS_END, +}; + +static const struct fds_xml_args connection[] = { + FDS_OPTS_NESTED(ENDPOINTS, "endpoints", endpoints, 0), + FDS_OPTS_ELEM (USER, "user", FDS_OPTS_T_STRING, 0), + FDS_OPTS_ELEM (PASSWORD, "password", FDS_OPTS_T_STRING, 0), + FDS_OPTS_ELEM (DATABASE, "database", FDS_OPTS_T_STRING, 0), + FDS_OPTS_ELEM (TABLE, "table", FDS_OPTS_T_STRING, 0), + FDS_OPTS_END, +}; + +static const struct fds_xml_args root[] = { + FDS_OPTS_ROOT ("params"), + FDS_OPTS_NESTED(CONNECTION, "connection", connection, 0), + FDS_OPTS_ELEM (INSERTER_THREADS, "inserterThreads", FDS_OPTS_T_UINT, FDS_OPTS_P_OPT), + FDS_OPTS_ELEM (BLOCKS, "blocks", FDS_OPTS_T_UINT, FDS_OPTS_P_OPT), + FDS_OPTS_ELEM (BLOCK_INSERT_THRESHOLD, "blockInsertThreshold", FDS_OPTS_T_UINT, FDS_OPTS_P_OPT), + FDS_OPTS_ELEM (BLOCK_INSERT_MAX_DELAY_SECS, "blockInsertMaxDelaySecs", FDS_OPTS_T_UINT, FDS_OPTS_P_OPT), + FDS_OPTS_ELEM (SPLIT_BIFLOW, "splitBiflow", FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT), + FDS_OPTS_ELEM (BIFLOW_EMPTY_AUTOIGNORE, "biflowEmptyAutoignore", FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT), + FDS_OPTS_ELEM (NONBLOCKING, "nonblocking", FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT), + FDS_OPTS_NESTED(COLUMNS, "columns", columns, 0), + FDS_OPTS_END, +}; + +} + +static std::optional parse_special_field(const std::string &name) +{ + if (name == "odid") { + return {SpecialField::ODID}; + } + return {}; +} + +static Config::Column parse_column(fds_xml_ctx_t *column_ctx, const fds_iemgr_t *iemgr) +{ + const fds_xml_cont *content; + Config::Column column; + std::string source; + + while (fds_xml_next(column_ctx, &content) == FDS_OK) { + if (content->id == args::NAME) { + column.name = content->ptr_string; + } else if (content->id == args::NULLABLE) { + column.nullable = content->val_bool; + } else if (content->id == args::SOURCE) { + source = content->ptr_string; + } + } + + if (source.empty()) { + source = column.name; + } + + const fds_iemgr_elem *elem = fds_iemgr_elem_find_name(iemgr, source.c_str()); + const fds_iemgr_alias *alias = fds_iemgr_alias_find(iemgr, source.c_str()); + std::optional special = parse_special_field(source); + if (!special && !elem && !alias) { + throw std::runtime_error("IPFIX element with name \"" + source + "\" not found"); + } else if (special) { + column.source = *special; + } else if (alias) { + column.source = alias; + } else if (elem) { + column.source = elem; + } + + return column; +} + +static std::vector parse_columns(fds_xml_ctx_t *columns_ctx, const fds_iemgr_t *iemgr) +{ + const fds_xml_cont *content; + std::vector columns; + + while (fds_xml_next(columns_ctx, &content) == FDS_OK) { + columns.push_back(parse_column(content->ptr_ctx, iemgr)); + } + + return columns; +} + +static Config::Endpoint parse_endpoint(fds_xml_ctx_t *endpoint_ctx) +{ + Config::Endpoint endpoint; + const fds_xml_cont *content; + + while (fds_xml_next(endpoint_ctx, &content) == FDS_OK) { + if (content->id == args::HOST) { + endpoint.host = content->ptr_string; + + } else if (content->id == args::PORT) { + if (content->val_uint > std::numeric_limits::max()) { + throw std::runtime_error(std::to_string(content->val_uint) + " is not a valid port number"); + } + endpoint.port = content->val_uint; + + } + } + + return endpoint; +} + +static std::vector parse_endpoints(fds_xml_ctx_t *endpoints_ctx) +{ + std::vector endpoints; + const fds_xml_cont *content; + + while (fds_xml_next(endpoints_ctx, &content) == FDS_OK) { + if (content->id == args::ENDPOINT) { + endpoints.push_back(parse_endpoint(content->ptr_ctx)); + } + } + + return endpoints; +} + +static Config::Connection parse_connection(fds_xml_ctx_t *connection_ctx) +{ + Config::Connection connection; + const fds_xml_cont *content; + + while (fds_xml_next(connection_ctx, &content) == FDS_OK) { + if (content->id == args::USER) { + connection.user = content->ptr_string; + + } else if (content->id == args::PASSWORD) { + connection.password = content->ptr_string; + + } else if (content->id == args::DATABASE) { + connection.database = content->ptr_string; + + } else if (content->id == args::TABLE) { + connection.table = content->ptr_string; + + } else if (content->id == args::ENDPOINTS) { + connection.endpoints = parse_endpoints(content->ptr_ctx); + } + } + + return connection; +} + +static void parse_root(fds_xml_ctx_t *root_ctx, const fds_iemgr_t *iemgr, Config &config) +{ + const fds_xml_cont *content; + + while (fds_xml_next(root_ctx, &content) == FDS_OK) { + if (content->id == args::CONNECTION) { + config.connection = parse_connection(content->ptr_ctx); + + } else if (content->id == args::COLUMNS) { + config.columns = parse_columns(content->ptr_ctx, iemgr); + + } else if (content->id == args::BLOCKS) { + config.blocks = content->val_uint; + + } else if (content->id == args::INSERTER_THREADS) { + config.inserter_threads = content->val_uint; + + } else if (content->id == args::BLOCK_INSERT_THRESHOLD) { + config.block_insert_threshold = content->val_uint; + + } else if (content->id == args::BLOCK_INSERT_MAX_DELAY_SECS) { + config.block_insert_max_delay_secs = content->val_uint; + + } else if (content->id == args::SPLIT_BIFLOW) { + config.split_biflow = content->val_bool; + + } else if (content->id == args::BIFLOW_EMPTY_AUTOIGNORE) { + config.biflow_empty_autoignore = content->val_bool; + + } else if (content->id == args::NONBLOCKING) { + config.nonblocking = content->val_bool; + + } + } +} + +Config parse_config(const char *xml_string, const fds_iemgr_t *iemgr) +{ + Config config{}; + + std::unique_ptr parser(fds_xml_create(), &fds_xml_destroy); + if (!parser) { + throw std::runtime_error("Failed to create an XML parser!"); + } + + if (fds_xml_set_args(parser.get(), args::root) != FDS_OK) { + std::string err = fds_xml_last_err(parser.get()); + throw std::runtime_error("Failed to parse the description of an XML document: " + err); + } + + fds_xml_ctx_t *root_ctx = fds_xml_parse_mem(parser.get(), xml_string, true); + if (!root_ctx) { + std::string err = fds_xml_last_err(parser.get()); + throw std::runtime_error("Failed to parse the configuration: " + err); + } + + parse_root(root_ctx, iemgr, config); + + return config; +} diff --git a/extra_plugins/output/clickhouse/src/config.h b/extra_plugins/output/clickhouse/src/config.h new file mode 100644 index 000000000..6ca8c40ef --- /dev/null +++ b/extra_plugins/output/clickhouse/src/config.h @@ -0,0 +1,67 @@ +/** + * @file + * @author Michal Sedlak + * @brief Configuration parsing and representation + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include + +#include +#include +#include +#include + +enum class SpecialField { + NONE = 0, + ODID, +}; + +/** + * @class Config + * @brief A struct containing all the configurable plugin parameters + */ +struct Config { + struct Column { + std::string name; + bool nullable = false; + std::variant source; + }; + + struct Endpoint { + std::string host; + uint16_t port = 9000; + }; + + struct Connection { + std::vector endpoints; + std::string user; + std::string password; + std::string database; + std::string table; + }; + + Connection connection; + std::vector columns; + uint64_t inserter_threads = 8; + uint64_t blocks = 64; + uint64_t block_insert_threshold = 100000; + uint64_t block_insert_max_delay_secs = 10; + bool split_biflow = true; + bool biflow_empty_autoignore = true; + bool nonblocking = true; +}; + +/** + * @brief Parse a XML config into a structured form + * + * @param xml The config as a XML string + * @param iemgr The iemgr instance + * @return The parsed config + */ +Config parse_config(const char *xml, const fds_iemgr_t *iemgr); From 7598e6758f8393c2fb457037d011de77a4fd2e8c Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 14 Nov 2024 12:25:54 +0100 Subject: [PATCH 04/40] Clickhouse - introduce clickhouse include wrapper --- extra_plugins/output/clickhouse/src/clickhouse.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 extra_plugins/output/clickhouse/src/clickhouse.h diff --git a/extra_plugins/output/clickhouse/src/clickhouse.h b/extra_plugins/output/clickhouse/src/clickhouse.h new file mode 100644 index 000000000..835ced0ef --- /dev/null +++ b/extra_plugins/output/clickhouse/src/clickhouse.h @@ -0,0 +1,14 @@ +/** + * @file + * @author Michal Sedlak + * @brief ClickHouse client library wrapper + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +#include +#pragma GCC diagnostic pop From b82041a2525afc1139d30c1384fceae652e76bd5 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 14 Nov 2024 12:25:34 +0100 Subject: [PATCH 05/40] Clickhouse - introduce datatype --- .../output/clickhouse/src/datatype.cpp | 414 ++++++++++++++++++ .../output/clickhouse/src/datatype.h | 220 ++++++++++ 2 files changed, 634 insertions(+) create mode 100644 extra_plugins/output/clickhouse/src/datatype.cpp create mode 100644 extra_plugins/output/clickhouse/src/datatype.h diff --git a/extra_plugins/output/clickhouse/src/datatype.cpp b/extra_plugins/output/clickhouse/src/datatype.cpp new file mode 100644 index 000000000..f22f2f346 --- /dev/null +++ b/extra_plugins/output/clickhouse/src/datatype.cpp @@ -0,0 +1,414 @@ +/** + * @file + * @author Michal Sedlak + * @brief Functions specific to individual data types + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "datatype.h" +#include "common.h" + +template class ColumnDateTime64 : public clickhouse::ColumnDateTime64 { +public: + ColumnDateTime64() : clickhouse::ColumnDateTime64(Precision) {} +}; + +DataType type_from_ipfix(fds_iemgr_element_type type) +{ + switch (type) { + case FDS_ET_STRING: return DataType::String; + case FDS_ET_SIGNED_8: return DataType::Int8; + case FDS_ET_SIGNED_16: return DataType::Int16; + case FDS_ET_SIGNED_32: return DataType::Int32; + case FDS_ET_SIGNED_64: return DataType::Int64; + case FDS_ET_UNSIGNED_8: return DataType::UInt8; + case FDS_ET_UNSIGNED_16: return DataType::UInt16; + case FDS_ET_UNSIGNED_32: return DataType::UInt32; + case FDS_ET_UNSIGNED_64: return DataType::UInt64; + case FDS_ET_IPV4_ADDRESS: return DataType::IPv4; + case FDS_ET_IPV6_ADDRESS: return DataType::IPv6; + case FDS_ET_DATE_TIME_SECONDS: return DataType::DatetimeSecs; + case FDS_ET_DATE_TIME_MILLISECONDS: return DataType::DatetimeMillisecs; + case FDS_ET_DATE_TIME_MICROSECONDS: return DataType::DatetimeMicrosecs; + case FDS_ET_DATE_TIME_NANOSECONDS: return DataType::DatetimeNanosecs; + case FDS_ET_MAC_ADDRESS: return DataType::Mac; + case FDS_ET_FLOAT_32: return DataType::Float32; + case FDS_ET_FLOAT_64: return DataType::Float64; + case FDS_ET_OCTET_ARRAY: return DataType::OctetArray; + default: throw Error("unsupported IPFIX data type {}", type); + } +} + +static DataType unify_type(DataType a, DataType b) +{ + auto is_int = [](DataType t) { + return t == DataType::Int8 + || t == DataType::Int16 + || t == DataType::Int32 + || t == DataType::Int64; + }; + auto is_uint = [](DataType t) { + return t == DataType::UInt8 + || t == DataType::UInt16 + || t == DataType::UInt32 + || t == DataType::UInt64; + }; + auto is_ip = [](DataType t) { + return t == DataType::IPv4 + || t == DataType::IPv6 + || t == DataType::IP; + }; + auto is_datetime = [](DataType t) { + return t == DataType::DatetimeSecs + || t == DataType::DatetimeMillisecs + || t == DataType::DatetimeMicrosecs + || t == DataType::DatetimeNanosecs; + }; + auto is_float = [](DataType t) { + return t == DataType::Float32 + || t == DataType::Float64; + }; + + if (a == b) { + return a; + } + if (is_int(a) && is_int(b)) { + return std::max(a, b); + } + if (is_uint(a) && is_uint(b)) { + return std::max(a, b); + } + if (is_datetime(a) && is_datetime(b)) { + return std::max(a, b); + } + if (is_float(a) && is_float(b)) { + return std::max(a, b); + } + if (is_ip(a) && is_ip(b)) { + return DataType::IP; + } + + throw Error("cannot unify types {} and {}", a, b); +} + +DataType find_common_type(const fds_iemgr_alias &alias) +{ + if (alias.sources_cnt == 0) { + throw Error("alias \"{}\" has no sources", alias.name); + } + + DataType common_type = type_from_ipfix(alias.sources[0]->data_type); + for (size_t i = 1; i < alias.sources_cnt; i++) { + common_type = unify_type(common_type, type_from_ipfix(alias.sources[i]->data_type)); + } + return common_type; +} + +namespace getters { + +template +static UIntType get_uint(fds_drec_field field) +{ + uint64_t value = 0; + int ret = fds_get_uint_be(field.data, field.size, &value); + if (ret != FDS_OK) { + throw Error("fds_get_uint_be() has failed: {}", ret); + } + return static_cast(value); +} + +template +static IntType get_int(fds_drec_field field) +{ + int64_t value = 0; + int ret = fds_get_int_be(field.data, field.size, &value); + if (ret != FDS_OK) { + throw Error("fds_get_int_be() has failed: {}", ret); + } + return static_cast(value); +} + +static IP4Addr get_ipv4(fds_drec_field field) +{ + IP4Addr value; + int ret = fds_get_ip(field.data, field.size, &value.s_addr); + if (ret != FDS_OK) { + throw Error("fds_get_ip() has failed: {}", ret); + } + return value; +} + +static IP6Addr get_ipv6(fds_drec_field field) +{ + IP6Addr value; + int ret = fds_get_ip(field.data, field.size, &value.s6_addr); + if (ret != FDS_OK) { + throw Error("fds_get_ip() has failed: {}", ret); + } + return value; +} + +static IP6Addr get_ip(fds_drec_field field) +{ + IP6Addr value; + int ret; + if (field.size == 4) { + static constexpr uint8_t IPV4_MAPPED_IPV6_PREFIX[]{ + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xFF, 0xFF}; + std::memcpy( + reinterpret_cast(&value.s6_addr), + IPV4_MAPPED_IPV6_PREFIX, + sizeof(IPV4_MAPPED_IPV6_PREFIX)); + ret = fds_get_ip(field.data, field.size, &reinterpret_cast(&value.s6_addr)[12]); + } else { + ret = fds_get_ip(field.data, field.size, &value.s6_addr); + } + if (ret != FDS_OK) { + throw Error("fds_get_ip() has failed: {}", ret); + } + return value; +} + +static std::string_view get_string(fds_drec_field field) +{ + return std::string_view(reinterpret_cast(field.data), field.size); +} + +static std::string_view get_octetarray(fds_drec_field field) +{ + return std::string_view(reinterpret_cast(field.data), field.size); +} + +static uint64_t get_datetime(fds_drec_field field) +{ + uint64_t value = 0; + int ret = fds_get_datetime_lp_be(field.data, field.size, field.info->def->data_type, &value); + if (ret != FDS_OK) { + throw Error("fds_get_datetime_lp_be() has failed: {}", ret); + } + value /= 1000; + return value; +} + +template +static int64_t get_datetime64(fds_drec_field field) +{ + int64_t value = 0; + timespec ts; + int ret = fds_get_datetime_hp_be(field.data, field.size, field.info->def->data_type, &ts); + if (ret != FDS_OK) { + throw Error("fds_get_datetime_hp_be() has failed: {}", ret); + } + value = (static_cast(ts.tv_sec) * 1'000'000'000 + static_cast(ts.tv_nsec)) / Divisor; + return value; +} + +static uint64_t get_mac(fds_drec_field field) +{ + uint64_t value = 0; + fds_get_mac(field.data, field.size, reinterpret_cast(&value)); + return value; +} + +template +static FloatType get_float(fds_drec_field field) +{ + double value = 0; + int ret = fds_get_float_be(field.data, field.size, &value); + if (ret != FDS_OK) { + throw Error("fds_get_float_be() has failed: {}", ret); + } + return static_cast(value); +} +} + +std::string type_to_clickhouse(DataType type, bool nullable) +{ + if (nullable) { + return "Nullable(" + type_to_clickhouse(type, false) + ")"; + } + + switch (type) { + case DataType::UInt8: return "UInt8"; + case DataType::UInt16: return "UInt16"; + case DataType::UInt32: return "UInt32"; + case DataType::UInt64: return "UInt64"; + case DataType::Int8: return "Int8"; + case DataType::Int16: return "Int16"; + case DataType::Int32: return "Int32"; + case DataType::Int64: return "Int64"; + case DataType::IP: return "IPv6"; + case DataType::IPv4: return "IPv4"; + case DataType::IPv6: return "IPv6"; + case DataType::String: return "String"; + case DataType::DatetimeSecs: return "DateTime"; + case DataType::DatetimeMillisecs: return "DateTime64(3)"; + case DataType::DatetimeMicrosecs: return "DateTime64(6)"; + case DataType::DatetimeNanosecs: return "DateTime64(9)"; + case DataType::Mac: return "UInt64"; + case DataType::Float32: return "Float32"; + case DataType::Float64: return "Float64"; + case DataType::OctetArray: return "String"; + case DataType::Invalid: throw std::logic_error("invalid data type"); + } + + throw std::logic_error("unexpected datatype value"); +} + +std::shared_ptr make_column(DataType type, bool nullable) +{ + if (nullable) { + switch (type) { + case DataType::UInt8: return std::make_shared>(); + case DataType::UInt16: return std::make_shared>(); + case DataType::UInt32: return std::make_shared>(); + case DataType::UInt64: return std::make_shared>(); + case DataType::Int8: return std::make_shared>(); + case DataType::Int16: return std::make_shared>(); + case DataType::Int32: return std::make_shared>(); + case DataType::Int64: return std::make_shared>(); + case DataType::IP: return std::make_shared>(); + case DataType::IPv4: return std::make_shared>(); + case DataType::IPv6: return std::make_shared>(); + case DataType::String: return std::make_shared>(); + case DataType::DatetimeSecs: return std::make_shared>(); + case DataType::DatetimeMillisecs: return std::make_shared>>(); + case DataType::DatetimeMicrosecs: return std::make_shared>>(); + case DataType::DatetimeNanosecs: return std::make_shared>>(); + case DataType::Mac: return std::make_shared>(); + case DataType::Float32: return std::make_shared>(); + case DataType::Float64: return std::make_shared>(); + case DataType::OctetArray: return std::make_shared>(); + case DataType::Invalid: throw std::logic_error("invalid data type"); + } + } else { + switch (type) { + case DataType::UInt8: return std::make_shared(); + case DataType::UInt16: return std::make_shared(); + case DataType::UInt32: return std::make_shared(); + case DataType::UInt64: return std::make_shared(); + case DataType::Int8: return std::make_shared(); + case DataType::Int16: return std::make_shared(); + case DataType::Int32: return std::make_shared(); + case DataType::Int64: return std::make_shared(); + case DataType::IP: return std::make_shared(); + case DataType::IPv4: return std::make_shared(); + case DataType::IPv6: return std::make_shared(); + case DataType::String: return std::make_shared(); + case DataType::DatetimeSecs: return std::make_shared(); + case DataType::DatetimeMillisecs: return std::make_shared>(); + case DataType::DatetimeMicrosecs: return std::make_shared>(); + case DataType::DatetimeNanosecs: return std::make_shared>(); + case DataType::Mac: return std::make_shared(); + case DataType::Float32: return std::make_shared(); + case DataType::Float64: return std::make_shared(); + case DataType::OctetArray: return std::make_shared(); + case DataType::Invalid: throw std::logic_error("invalid data type"); + } + } + + throw std::logic_error("unexpected datatype value"); +} + +ValueVariant get_value(DataType type, fds_drec_field& field) +{ + switch (type) { + case DataType::UInt8: return getters::get_uint(field); + case DataType::UInt16: return getters::get_uint(field); + case DataType::UInt32: return getters::get_uint(field); + case DataType::UInt64: return getters::get_uint(field); + case DataType::Int8: return getters::get_int(field); + case DataType::Int16: return getters::get_int(field); + case DataType::Int32: return getters::get_int(field); + case DataType::Int64: return getters::get_int(field); + case DataType::IP: return getters::get_ip(field); + case DataType::IPv4: return getters::get_ipv4(field); + case DataType::IPv6: return getters::get_ipv6(field); + case DataType::String: return getters::get_string(field); + case DataType::DatetimeSecs: return getters::get_datetime(field); + case DataType::DatetimeMillisecs: return getters::get_datetime64<1'000'000>(field); + case DataType::DatetimeMicrosecs: return getters::get_datetime64<1'000>(field); + case DataType::DatetimeNanosecs: return getters::get_datetime64<1>(field); + case DataType::Mac: return getters::get_mac(field); + case DataType::Float32: return getters::get_float(field); + case DataType::Float64: return getters::get_float(field); + case DataType::OctetArray: return getters::get_octetarray(field); + case DataType::Invalid: throw std::logic_error("invalid data type"); + } + + throw std::logic_error("unexpected datatype value"); +} + +template +static inline void column_append_nullable(clickhouse::Column& column, ValueVariant* value) +{ + auto *concrete_col = reinterpret_cast *>(&column); + concrete_col->Append(value ? std::optional{std::get(*value)} : std::nullopt); +} + +template +static inline void column_append(clickhouse::Column& column, ValueVariant* value) +{ + auto *concrete_col = reinterpret_cast(&column); + concrete_col->Append(value ? std::get(*value) : ValueT{}); +} + +void write_to_column(DataType type, bool nullable, clickhouse::Column& column, ValueVariant* value) +{ + if (nullable) { + switch (type) { + case DataType::UInt8: column_append_nullable(column, value); return; + case DataType::UInt16: column_append_nullable(column, value); return; + case DataType::UInt32: column_append_nullable(column, value); return; + case DataType::UInt64: column_append_nullable(column, value); return; + case DataType::Int8: column_append_nullable(column, value); return; + case DataType::Int16: column_append_nullable(column, value); return; + case DataType::Int32: column_append_nullable(column, value); return; + case DataType::Int64: column_append_nullable(column, value); return; + case DataType::IP: column_append_nullable(column, value); return; + case DataType::IPv4: column_append_nullable(column, value); return; + case DataType::IPv6: column_append_nullable(column, value); return; + case DataType::String: column_append_nullable(column, value); return; + case DataType::DatetimeSecs: column_append_nullable(column, value); return; + case DataType::DatetimeMillisecs: column_append_nullable, int64_t>(column, value); return; + case DataType::DatetimeMicrosecs: column_append_nullable, int64_t>(column, value); return; + case DataType::DatetimeNanosecs: column_append_nullable, int64_t>(column, value); return; + case DataType::Mac: column_append_nullable(column, value); return; + case DataType::Float32: column_append_nullable(column, value); return; + case DataType::Float64: column_append_nullable(column, value); return; + case DataType::OctetArray: column_append_nullable(column, value); return; + case DataType::Invalid: throw std::logic_error("invalid data type"); + } + } else { + switch (type) { + case DataType::UInt8: column_append(column, value); return; + case DataType::UInt16: column_append(column, value); return; + case DataType::UInt32: column_append(column, value); return; + case DataType::UInt64: column_append(column, value); return; + case DataType::Int8: column_append(column, value); return; + case DataType::Int16: column_append(column, value); return; + case DataType::Int32: column_append(column, value); return; + case DataType::Int64: column_append(column, value); return; + case DataType::IP: column_append(column, value); return; + case DataType::IPv4: column_append(column, value); return; + case DataType::IPv6: column_append(column, value); return; + case DataType::String: column_append(column, value); return; + case DataType::DatetimeSecs: column_append(column, value); return; + case DataType::DatetimeMillisecs: column_append, int64_t>(column, value); return; + case DataType::DatetimeMicrosecs: column_append, int64_t>(column, value); return; + case DataType::DatetimeNanosecs: column_append, int64_t>(column, value); return; + case DataType::Mac: column_append(column, value); return; + case DataType::Float32: column_append(column, value); return; + case DataType::Float64: column_append(column, value); return; + case DataType::OctetArray: column_append(column, value); return; + case DataType::Invalid: throw std::logic_error("invalid data type"); + } + } + + throw std::logic_error("unexpected datatype value"); +} diff --git a/extra_plugins/output/clickhouse/src/datatype.h b/extra_plugins/output/clickhouse/src/datatype.h new file mode 100644 index 000000000..10338e140 --- /dev/null +++ b/extra_plugins/output/clickhouse/src/datatype.h @@ -0,0 +1,220 @@ +/** + * @file + * @author Michal Sedlak + * @brief Functions specific to individual data types + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "clickhouse.h" + +#include +#include + +#include +#include +#include + + +/** An intermediary data type used for conversions between IPFIX and ClickHouse types */ +enum class DataType { + Invalid, + + // Lowest to highest precision. Keep it that way! + // This is used to simplify upcasting in the unify_type function. + // e.g. std::max(Int8, Int16) = Int16 + Int8, + Int16, + Int32, + Int64, + + UInt8, + UInt16, + UInt32, + UInt64, + + Float32, + Float64, + + IP, // IP address regardless of version + IPv4, + IPv6, + + String, + OctetArray, + + DatetimeSecs, + DatetimeMillisecs, + DatetimeMicrosecs, + DatetimeNanosecs, + + Mac, +}; + +/** + * @brief Simple in_addr wrapper + */ +struct IP4Addr : public in_addr { + IP4Addr() { + s_addr = 0; + } + + IP4Addr(in_addr addr) : in_addr(addr) {} + + friend bool operator==(const IP4Addr &a, const IP4Addr &b) { + return a.s_addr == b.s_addr; + } + + friend bool operator!=(const IP4Addr &a, const IP4Addr &b) { + return !(a == b); + } +}; + +/** + * @brief Simple in6_addr wrapper + */ +struct IP6Addr : public in6_addr { + IP6Addr() { + std::memset(s6_addr, 0, sizeof(s6_addr)); + } + + IP6Addr(in6_addr addr) : in6_addr(addr) {} + + friend bool operator==(const IP6Addr &a, const IP6Addr &b) { + return std::memcmp(&a.s6_addr, &b.s6_addr, sizeof(a.s6_addr)) == 0; + } + + friend bool operator!=(const IP6Addr &a, const IP6Addr &b) { + return !(a == b); + } +}; + + +/** Formatter for the DataType enum */ +template <> struct fmt::formatter : formatter { + auto format(DataType type, format_context& ctx) const -> format_context::iterator { + const std::string_view name = [=]() { + switch (type) { + case DataType::Invalid: return "Invalid"; + case DataType::Int8: return "Int8"; + case DataType::Int16: return "Int16"; + case DataType::Int32: return "Int32"; + case DataType::Int64: return "Int64"; + case DataType::UInt8: return "UInt8"; + case DataType::UInt16: return "UInt16"; + case DataType::UInt32: return "UInt32"; + case DataType::UInt64: return "UInt64"; + case DataType::IP: return "IP"; + case DataType::IPv4: return "IPv4"; + case DataType::IPv6: return "IPv6"; + case DataType::String: return "String"; + case DataType::DatetimeNanosecs: return "DatetimeNanosecs"; + case DataType::DatetimeMicrosecs: return "DatetimeMicrosecs"; + case DataType::DatetimeMillisecs: return "DatetimeMillisecs"; + case DataType::DatetimeSecs: return "DatetimeSecs"; + case DataType::Mac: return "Mac"; + case DataType::Float32: return "Float32"; + case DataType::Float64: return "Float64"; + case DataType::OctetArray: return "OctetArray"; + } + return "???"; + }(); + return formatter::format(name, ctx); + } +}; + +/** Formatter for the iemgr element type */ +template <> struct fmt::formatter : formatter { + auto format(fds_iemgr_element_type type, format_context& ctx) const -> format_context::iterator { + const char *name = fds_iemgr_type2str(type); + if (!name) { + name = "???"; + } + return formatter::format(name, ctx); + } +}; + +/** + * @brief Get intermediary data type for a corresponding IPFIX element type + * + * @param type The IPFIX type + * @return The intermediary data type + */ +DataType type_from_ipfix(fds_iemgr_element_type type); + +/** + * @brief Get ClickHouse data type for the intermediary data type + * + * @param type The intermediary data type + * @param nullable Whether the type is nullable or not + * @return The ClickHouse data type + */ +std::string type_to_clickhouse(DataType type, bool nullable); + +/** + * @brief Find an intermediary data type that can be used to store all the possible data types of the alias + * + * @param alias The iemgr alias + * @return The common data type + */ +DataType find_common_type(const fds_iemgr_alias &alias); + + +using ValueVariant = std::variant< + std::monostate, + uint8_t, + uint16_t, + uint32_t, + uint64_t, + int8_t, + int16_t, + int32_t, + int64_t, + IP4Addr, + IP6Addr, + float, + double, + std::string_view +>; + +/** + * @brief Make a ClickHouse column that is able to store values of the supplied data type + * + * @param type The data type + * @param nullable Whether the type is nullable or not + * @return The ClickHouse column object + */ +std::shared_ptr make_column(DataType type, bool nullable); + +/** + * @brief An error thrown when the conversion from IPFIX to ClickHouse representation fails. + */ +class ConversionError : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +/** + * Retrieves the value of a specific data type from the given field. + * + * @param type The data type to retrieve. + * @param field The field from which the value is extracted. + * @return A variant containing the value of the specified data type. + * + * @throws ConversionError if the conversion fails. + */ +ValueVariant get_value(DataType type, fds_drec_field& field); + +/** + * Writes a value to a ClickHouse column. + * + * @param type The data type of the value. + * @param nullable Indicates whether the column allows null values. + * @param column The ClickHouse column to which the value is written. + * @param value A pointer to the value to be written. + */ +void write_to_column(DataType type, bool nullable, clickhouse::Column& column, ValueVariant* value); From 68e33b7a7b47a9e472525171867449ebfc681317 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 14 Nov 2024 12:25:11 +0100 Subject: [PATCH 06/40] Clickhouse - introduce common --- .../output/clickhouse/src/common.cpp | 21 ++++++ extra_plugins/output/clickhouse/src/common.h | 68 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 extra_plugins/output/clickhouse/src/common.cpp create mode 100644 extra_plugins/output/clickhouse/src/common.h diff --git a/extra_plugins/output/clickhouse/src/common.cpp b/extra_plugins/output/clickhouse/src/common.cpp new file mode 100644 index 000000000..c33803233 --- /dev/null +++ b/extra_plugins/output/clickhouse/src/common.cpp @@ -0,0 +1,21 @@ +/** + * @file + * @author Michal Sedlak + * @brief Common functionality and general utilities + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "common.h" + +#include + +std::string to_lower(std::string str) +{ + for (auto &c : str) { + c = std::tolower(c); + } + return str; +} diff --git a/extra_plugins/output/clickhouse/src/common.h b/extra_plugins/output/clickhouse/src/common.h new file mode 100644 index 000000000..eefb5166a --- /dev/null +++ b/extra_plugins/output/clickhouse/src/common.h @@ -0,0 +1,68 @@ +/** + * @file + * @author Michal Sedlak + * @brief Common functionality and general utilities + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ +#pragma once + +#include "clickhouse.h" + +#include +#include + +#include + +/** + * @class Nonmoveable + * @brief Classes that will inherit from this class will be immovable + */ +struct Nonmoveable { + Nonmoveable() = default; + Nonmoveable(Nonmoveable&&) = delete; + Nonmoveable& operator=(Nonmoveable&&) = delete; +}; + +/** + * @class Noncopyable + * @brief Classes that will inherit from this class will not be able to be copied + */ +struct Noncopyable { + Noncopyable() = default; + Noncopyable(const Noncopyable&) = delete; + Noncopyable& operator=(const Noncopyable&) = delete; +}; + +/** + * @class Error + * @brief An exception type that will be thrown by the plugin + */ +class Error : public std::runtime_error { +public: + template + Error(Args&& ...args) : std::runtime_error(fmt::format(args...)) {} +}; + + +class Logger { +public: + Logger(const ipx_ctx_t *ctx) : m_ctx(ctx) {} + template void info (const char *fmt, Args... args) { IPX_CTX_INFO (m_ctx, fmt, args..., 0); } + template void warning(const char *fmt, Args... args) { IPX_CTX_WARNING(m_ctx, fmt, args..., 0); } + template void error (const char *fmt, Args... args) { IPX_CTX_ERROR (m_ctx, fmt, args..., 0); } + template void debug (const char *fmt, Args... args) { IPX_CTX_DEBUG (m_ctx, fmt, args..., 0); } + +private: + const ipx_ctx_t *m_ctx; +}; + + +/** + * @brief Convert string to lowercase + * + * @param str The string + */ +std::string to_lower(std::string s); From 39adf38559895cb0322186d92b161fef0acd5dc1 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 9 Jan 2025 13:46:09 +0100 Subject: [PATCH 07/40] Clickhouse - introduce SyncQueue class --- .../output/clickhouse/src/syncqueue.h | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 extra_plugins/output/clickhouse/src/syncqueue.h diff --git a/extra_plugins/output/clickhouse/src/syncqueue.h b/extra_plugins/output/clickhouse/src/syncqueue.h new file mode 100644 index 000000000..b978706ce --- /dev/null +++ b/extra_plugins/output/clickhouse/src/syncqueue.h @@ -0,0 +1,89 @@ +/** + * @file + * @author Michal Sedlak + * @brief Thread-safe queue implementation + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ +#pragma once + +#include +#include +#include +#include +#include + +/** + * @brief A thread-safe queue + */ +template +class SyncQueue { +public: + /** + * @brief Put an item into the queue + * + * @param item The item + */ + void put(Item item) + { + std::lock_guard lock(m_mutex); + m_items.push(item); + m_size = m_items.size(); + m_avail_cv.notify_all(); + } + + /** + * @brief Get an item from the queue, block and wait if there aren't any + * + * @return The item + */ + Item get() + { + std::unique_lock lock(m_mutex); + while (true) { + if (!m_items.empty()) { + auto item = m_items.front(); + m_items.pop(); + m_size = m_items.size(); + return item; + } + m_avail_cv.wait(lock); + } + } + + /** + * @brief Try to get an item from the queue, return immediately if there aren't any + * + * @return The item or std::nullopt + */ + std::optional try_get() + { + std::unique_lock lock(m_mutex); + if (!m_items.empty()) { + auto item = m_items.front(); + m_items.pop(); + m_size = m_items.size(); + return item; + } else { + return std::nullopt; + } + } + + /** + * @brief Get the current size of the queue + * + * @return The number of items in the queue + */ + std::size_t size() + { + return m_size; + } + +private: + std::atomic_size_t m_size = 0; + std::queue m_items; + std::mutex m_mutex; + std::condition_variable m_avail_cv; +}; From 2958121f56a2fd6483197783ba2e067c98a5eb7e Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 10 Apr 2025 17:15:24 +0200 Subject: [PATCH 08/40] Clickhouse - introduce Stats --- extra_plugins/output/clickhouse/src/stats.cpp | 55 ++++++++++++++ extra_plugins/output/clickhouse/src/stats.h | 72 +++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 extra_plugins/output/clickhouse/src/stats.cpp create mode 100644 extra_plugins/output/clickhouse/src/stats.h diff --git a/extra_plugins/output/clickhouse/src/stats.cpp b/extra_plugins/output/clickhouse/src/stats.cpp new file mode 100644 index 000000000..419071c88 --- /dev/null +++ b/extra_plugins/output/clickhouse/src/stats.cpp @@ -0,0 +1,55 @@ +/** + * @file + * @author Michal Sedlak + * @brief Statistics tracking and reporting + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "stats.h" +#include "plugin.h" + +static constexpr int STATS_PRINT_INTERVAL_SECS = 1; + +Stats::Stats(Logger logger, Plugin& plugin) : m_logger(logger), m_plugin(plugin) {} + +void Stats::add_recs(uint64_t count) +{ + m_recs_processed_since_last += count; + m_recs_processed_total += count; +} + +void Stats::add_rows(uint64_t count) +{ + m_rows_written_total += count; +} + +void Stats::add_dropped(uint64_t count) +{ + m_recs_dropped_total += count; +} + +void Stats::print_stats_throttled(time_t now) +{ + if (m_start_time == 0) { + m_start_time = now; + } + + if ((now - m_last_stats_print_time) > STATS_PRINT_INTERVAL_SECS) { + double total_rps = m_recs_processed_total / std::max(1, now - m_start_time); + double immediate_rps = m_recs_processed_since_last / std::max(1, now - m_last_stats_print_time); + m_logger.info("STATS - RECS: %lu (%lu dropped), ROWS: %lu, AVG: %.2f recs/sec, AVG_IMMEDIATE: %.2f recs/sec, BLK_AVAIL_Q: %lu, BLK_FILL_Q: %lu", + m_recs_processed_total, + m_recs_dropped_total, + m_rows_written_total, + total_rps, + immediate_rps, + m_plugin.m_avail_blocks.size(), + m_plugin.m_filled_blocks.size() + ); + m_recs_processed_since_last = 0; + m_last_stats_print_time = now; + } +} diff --git a/extra_plugins/output/clickhouse/src/stats.h b/extra_plugins/output/clickhouse/src/stats.h new file mode 100644 index 000000000..60d562c9b --- /dev/null +++ b/extra_plugins/output/clickhouse/src/stats.h @@ -0,0 +1,72 @@ +/** + * @file + * @author Michal Sedlak + * @brief Statistics tracking and reporting + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "common.h" +#include +#include + +class Plugin; // Forward declaration of Plugin class. + +/** + * @class Stats + * @brief A class for tracking and reporting statistics related to data processing. + * + * The Stats class is responsible for maintaining counters and timestamps + * to track the progress of data processing and periodically printing + * statistics in a throttled manner. + */ +class Stats { +public: + /** + * @brief Constructs a Stats object. + * + * @param logger A Logger instance for logging statistics. + * @param plugin A reference to the Plugin instance associated with this Stats object. + */ + Stats(Logger logger, Plugin& plugin); + + /** + * @brief Adds a specified number of records to the processed count. + * + * @param count The number of records to add. + */ + void add_recs(uint64_t count); + + /** + * @brief Adds a specified number of rows to the written count. + * + * @param count The number of rows to add. + */ + void add_rows(uint64_t count); + + /** + * @brief Adds a specified number of records to the dropped count. + * + * @param count The number of records to add. + */ + void add_dropped(uint64_t count); + + /** + * @brief Prints the statistics if sufficient time has passed since the last print. + */ + void print_stats_throttled(time_t now); + +private: + Logger m_logger; ///< Logger instance for logging statistics. + Plugin &m_plugin; ///< Reference to the associated Plugin instance. + uint64_t m_rows_written_total = 0; ///< Total number of rows written. + uint64_t m_recs_processed_total = 0; ///< Total number of records processed. + uint64_t m_recs_processed_since_last = 0; ///< Records processed since the last statistics print. + uint64_t m_recs_dropped_total = 0; ///< Total number of records dropped. + time_t m_start_time = 0; ///< Start time of the statistics tracking. + time_t m_last_stats_print_time = 0; ///< Time of the last statistics print. +}; From 5b10cd567640291ba49c074b54fe65f4f75d5403 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 10 Apr 2025 17:15:10 +0200 Subject: [PATCH 09/40] Clickhouse - introduce Worker --- extra_plugins/output/clickhouse/src/worker.h | 122 +++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 extra_plugins/output/clickhouse/src/worker.h diff --git a/extra_plugins/output/clickhouse/src/worker.h b/extra_plugins/output/clickhouse/src/worker.h new file mode 100644 index 000000000..1425e060b --- /dev/null +++ b/extra_plugins/output/clickhouse/src/worker.h @@ -0,0 +1,122 @@ +/** + * @file + * @author Michal Sedlak + * @brief Worker class for running tasks in a separate thread + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "common.h" +#include + +/** + * @class Worker + * @brief A base class for creating worker threads that perform specific tasks. + * + * The Worker class provides a framework for running tasks in a separate thread, + * with support for stopping the task, checking for errors, and joining the thread. + * Derived classes must implement the `run` method to define the task logic. + */ +class Worker : Nonmoveable, Noncopyable { +public: + /** + * @brief Starts the worker thread and executes the task. + */ + void start() + { + m_thread = std::thread([this]() { this->do_run(); }); + } + + /** + * @brief Requests the worker to stop by setting a stop signal. + */ + void request_stop() + { + if (!m_stop_signal) { + m_stop_requested_at = std::time(nullptr); + } + m_stop_signal = true; + } + + /** + * @brief Checks if an error occurred during the task execution. + * + * @throws The exception captured during task execution, if any. + */ + void check_error() const + { + if (m_errored) { + std::rethrow_exception(m_exception); + } + } + + /** + * @brief Joins the worker thread if it is joinable. + */ + void join() + { + if (m_thread.joinable()) { + m_thread.join(); + } + } + + /** + * @brief Destructor that ensures the worker thread is joined. + */ + ~Worker() + { + join(); + } + +protected: + /** + * @brief Pure virtual method to define the task logic. + * + * Derived classes must implement this method to specify the task to be executed. + */ + virtual void run() = 0; + + /** + * @brief Checks if a stop request has been made. + * + * @return True if a stop request has been made, otherwise false. + */ + bool stop_requested() const + { + return m_stop_signal; + } + + /** + * Calculates the number of seconds that have passed since a stop was requested. + * + * @return The number of seconds elapsed since the stop request. + */ + unsigned int secs_since_stop_requested() const + { + return std::time(nullptr) - m_stop_requested_at; + } + +private: + std::thread m_thread; ///< The thread in which the task is executed. + std::atomic_bool m_stop_signal = false; ///< Signal to request the task to stop. + std::atomic_bool m_errored = false; ///< Flag indicating if an error occurred. + std::exception_ptr m_exception = nullptr; ///< Pointer to the captured exception, if any. + std::time_t m_stop_requested_at = 0; ///< When was stop requested. + + /** + * @brief Internal method to execute the task and handle exceptions. + */ + void do_run() + { + try { + run(); + } catch (...) { + m_exception = std::current_exception(); + m_errored = true; + } + } +}; From 02ec95643a8f592e6ef2fda78016ecbf885d5994 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 10 Apr 2025 17:14:28 +0200 Subject: [PATCH 10/40] Clickhouse - introduce Column --- extra_plugins/output/clickhouse/src/column.h | 31 ++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 extra_plugins/output/clickhouse/src/column.h diff --git a/extra_plugins/output/clickhouse/src/column.h b/extra_plugins/output/clickhouse/src/column.h new file mode 100644 index 000000000..41689addd --- /dev/null +++ b/extra_plugins/output/clickhouse/src/column.h @@ -0,0 +1,31 @@ +/** + * @file + * @author Michal Sedlak + * @brief Column representation + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "clickhouse.h" +#include "config.h" +#include "datatype.h" +#include +#include + +/** + * @class Column + * @brief Information about a column + */ +struct Column { + std::string name; + DataType datatype; + bool nullable; + + const fds_iemgr_elem *elem = nullptr; + const fds_iemgr_alias *alias = nullptr; + SpecialField special = SpecialField::NONE; +}; From 9f94e9676a22d295a7ab625d9239bb57f77fd84c Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 10 Apr 2025 17:13:49 +0200 Subject: [PATCH 11/40] Clickhouse - introduce Block --- extra_plugins/output/clickhouse/src/block.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 extra_plugins/output/clickhouse/src/block.h diff --git a/extra_plugins/output/clickhouse/src/block.h b/extra_plugins/output/clickhouse/src/block.h new file mode 100644 index 000000000..add67ce5c --- /dev/null +++ b/extra_plugins/output/clickhouse/src/block.h @@ -0,0 +1,20 @@ +/** + * @file + * @author Michal Sedlak + * @brief Block representation + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include +#include "clickhouse.h" + +struct Block { + std::vector> columns; + clickhouse::Block block; + unsigned int rows = 0; +}; From 3062375ebecc94f17e268483b64356e44a2f2bc2 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 10 Apr 2025 17:13:25 +0200 Subject: [PATCH 12/40] Clickhouse - introduce RecParser --- .../output/clickhouse/src/recparser.cpp | 204 ++++++++++++++++++ .../output/clickhouse/src/recparser.h | 146 +++++++++++++ 2 files changed, 350 insertions(+) create mode 100644 extra_plugins/output/clickhouse/src/recparser.cpp create mode 100644 extra_plugins/output/clickhouse/src/recparser.h diff --git a/extra_plugins/output/clickhouse/src/recparser.cpp b/extra_plugins/output/clickhouse/src/recparser.cpp new file mode 100644 index 000000000..43110dbfa --- /dev/null +++ b/extra_plugins/output/clickhouse/src/recparser.cpp @@ -0,0 +1,204 @@ +/** + * @file + * @author Michal Sedlak + * @brief Record parser + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "recparser.h" + +#include + +static int index_of_elem(const fds_iemgr_elem &elem, const fds_template &tmplt, bool rev) +{ + if (rev && !(tmplt.flags & FDS_TEMPLATE_BIFLOW)) { + return -1; + } + + const fds_tfield *fields = !rev ? tmplt.fields : tmplt.fields_rev; + for (std::size_t i = 0; i < tmplt.fields_cnt_total; i++) { + if (fields[i].en == elem.scope->pen && fields[i].id == elem.id) { + return i; + } + } + return -1; +} + +static int index_of_alias(const fds_iemgr_alias &alias, const fds_template &tmplt, bool rev) +{ + if (rev && !(tmplt.flags & FDS_TEMPLATE_BIFLOW)) { + return -1; + } + + for (std::size_t i = 0; i < alias.sources_cnt; i++) { + int idx = index_of_elem(*alias.sources[i], tmplt, rev); + if (idx != -1) { + return idx; + } + } + return -1; +} + +static bool is_skip(fds_drec_field &field, bool rev) +{ + static constexpr uint32_t IANA_EN = 0; + static constexpr uint32_t IANA_EN_REVERSE = 29305; + static constexpr uint16_t IANA_OCTET_DELTA_COUNT_ID = 1; + static constexpr uint16_t IANA_PACKET_DELTA_COUNT_ID = 2; + + uint32_t en = !rev ? IANA_EN : IANA_EN_REVERSE; + if (field.info->en == en && + (field.info->id == IANA_OCTET_DELTA_COUNT_ID || field.info->id == IANA_PACKET_DELTA_COUNT_ID) + ) { + uint64_t value; + if (fds_get_uint_be(field.data, field.size, &value) == FDS_OK && value == 0) { + return true; + } + } + return false; +} + +RecParser::RecParser(const std::vector &columns, const fds_template *tmplt, bool biflow_autoignore) +{ + m_tmplt.reset(fds_template_copy(tmplt)); + if (!m_tmplt) { + throw std::bad_alloc{}; + } + + m_biflow = tmplt->flags & FDS_TEMPLATE_BIFLOW; + m_biflow_autoignore = biflow_autoignore; + + m_fields.resize(columns.size(), fds_drec_field{nullptr, 0, nullptr}); + m_fields_rev.resize(columns.size(), fds_drec_field{nullptr, 0, nullptr}); + m_mapping.resize(tmplt->fields_cnt_total, -1); + m_mapping_rev.resize(tmplt->fields_cnt_total, -1); + + std::size_t column_idx = 0; + + for (const Column &column : columns) { + int field_idx = -1; + int rev_field_idx = -1; + + if (column.elem) { + field_idx = index_of_elem(*column.elem, *tmplt, false); + rev_field_idx = index_of_elem(*column.elem, *tmplt, true); + } else if (column.alias) { + field_idx = index_of_alias(*column.alias, *tmplt, false); + rev_field_idx = index_of_alias(*column.alias, *tmplt, true); + } + + if (field_idx != -1) { + assert(!column.elem || (tmplt->fields[field_idx].en == column.elem->scope->pen && + tmplt->fields[field_idx].id == column.elem->id)); + if (m_mapping[field_idx] == -1) { + m_mapping[field_idx] = column_idx; + } + } + + if (rev_field_idx != -1) { + assert(!column.elem || (tmplt->fields_rev[rev_field_idx].en == column.elem->scope->pen && + tmplt->fields_rev[rev_field_idx].id == column.elem->id)); + if (m_mapping_rev[rev_field_idx] == -1) { + m_mapping_rev[rev_field_idx] = column_idx; + } + } + + column_idx++; + } +} + +const fds_template *RecParser::tmplt() const +{ + return m_tmplt.get(); +} + +void RecParser::parse_record(fds_drec &rec) +{ + m_skip_flag_fwd = false; + m_skip_flag_rev = false; + + if (!m_biflow) { + m_skip_flag_rev = true; + } + + for (fds_drec_field &field : m_fields) { + field = fds_drec_field{nullptr, 0, nullptr}; + } + for (fds_drec_field &field : m_fields_rev) { + field = fds_drec_field{nullptr, 0, nullptr}; + } + + fds_drec_iter iter; + fds_drec_iter_init(&iter, &rec, 0); + int i; + while ((i = fds_drec_iter_next(&iter)) != FDS_EOC) { + if (m_biflow && m_biflow_autoignore) { + m_skip_flag_fwd |= is_skip(iter.field, false); + m_skip_flag_rev |= is_skip(iter.field, true); + } + + int field_idx = m_mapping[i]; + if (field_idx != -1) { + m_fields[field_idx] = iter.field; + } + + int rev_field_idx = m_mapping_rev[i]; + if (rev_field_idx != -1) { + m_fields_rev[rev_field_idx] = iter.field; + } + } +} + +fds_drec_field &RecParser::get_column(std::size_t idx, bool rev) +{ + assert((!rev && idx < m_fields.size()) || (rev && idx < m_fields_rev.size())); + assert(!rev || m_biflow); + return rev ? m_fields_rev[idx] : m_fields[idx]; +} + +bool RecParser::skip_fwd() const +{ + return m_skip_flag_fwd; +} + +bool RecParser::skip_rev() const +{ + return m_skip_flag_rev; +} + +RecParserManager::RecParserManager(const std::vector &columns, bool biflow_autoignore) + : m_columns(columns) + , m_biflow_autoignore(biflow_autoignore) +{} + +void RecParserManager::select_session(const ipx_session *sess) +{ + m_active_session = &m_sessions[sess]; +} + +void RecParserManager::select_odid(uint32_t odid) +{ + assert(m_active_session != nullptr); + m_active_odid = &(*m_active_session)[odid]; +} + +void RecParserManager::delete_session(const ipx_session *sess) +{ + m_sessions.erase(sess); + m_active_session = nullptr; + m_active_odid = nullptr; +} + +RecParser &RecParserManager::get_parser(const fds_template *tmplt) +{ + assert(m_active_session != nullptr && m_active_odid != nullptr); + auto it = m_active_odid->find(tmplt->id); + if (it == m_active_odid->end() || fds_template_cmp(it->second.tmplt(), tmplt) != 0) { + auto [it, _] = m_active_odid->insert_or_assign(tmplt->id, RecParser(m_columns, tmplt, m_biflow_autoignore)); + return it->second; + } + return it->second; +} diff --git a/extra_plugins/output/clickhouse/src/recparser.h b/extra_plugins/output/clickhouse/src/recparser.h new file mode 100644 index 000000000..8baad7d29 --- /dev/null +++ b/extra_plugins/output/clickhouse/src/recparser.h @@ -0,0 +1,146 @@ +/** + * @file + * @author Michal Sedlak + * @brief Record parser + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "column.h" +#include +#include +#include + +/** + * @class RecParser + * @brief Parses data records based on a given template and column configuration. + * + * The RecParser class is responsible for mapping fields from a data record + * to the corresponding columns, handling biflow templates, and determining + * whether certain records should be skipped. + */ +class RecParser { +public: + /** + * @brief Constructs a RecParser instance. + * + * @param columns A vector of Column objects. + * @param tmplt A pointer to the template used for parsing. + * @param biflow_autoignore Whether to automatically ignore empty biflow records. + */ + RecParser(const std::vector& columns, const fds_template *tmplt, bool biflow_autoignore); + + /** + * @brief Parses a data record and maps its fields to columns. + * + * @param rec The data record to parse. + */ + void parse_record(fds_drec &rec); + + /** + * @brief Retrieves the field corresponding to a specific column. + * + * If the field was not found in the record, the returned field points to + * NULL and has size of 0. + * + * If the provided index is outside of bounds, the behavior is undefined. + * + * @param idx The index of the column. + * @param rev Whether to retrieve the reverse field (for biflow templates). + * @return A reference to the corresponding field. + */ + fds_drec_field& get_column(std::size_t idx, bool rev = false); + + /** + * @brief Checks whether the forward direction of a biflow should be skipped. + * + * @return True if the forward direction should be skipped, otherwise false. + */ + bool skip_fwd() const; + + /** + * @brief Checks whether the reverse direction of a biflow should be skipped. + * + * @return True if the reverse direction should be skipped, otherwise false. + */ + bool skip_rev() const; + + const fds_template *tmplt() const; + +private: + struct tmplt_destroy { void operator()(fds_template *t) const { fds_template_destroy(t); } }; + + std::unique_ptr m_tmplt; + bool m_biflow; // Indicates whether the template is biflow. + bool m_biflow_autoignore; // Perform the skip checks. + bool m_skip_flag_fwd; // Skip the fwd direction of this biflow. + bool m_skip_flag_rev; // Skip the reverse direction of this biflow. + std::vector m_mapping; // Index of field in drec -> index of field in the field vec. + std::vector m_mapping_rev; // Index of field in drec -> index of field in the reverse field vec. + std::vector m_fields; // Field for the nth column. + std::vector m_fields_rev; // Reverse field for the nth column. +}; + +/** + * @class RecParserManager + * @brief Manages RecParser instances for different sessions, ODIDs, and templates. + * + * The RecParserManager class provides functionality to manage multiple RecParser + * instances, allowing selection of active sessions and ODIDs, and retrieval of + * parsers for specific templates. + */ +class RecParserManager { +public: + /** + * @brief Constructs a RecParserManager instance. + * + * @param columns A vector of Column objects defining the mapping. + * @param biflow_autoignore Whether to automatically ignore biflow records. + */ + RecParserManager(const std::vector &columns, bool biflow_autoignore); + + /** + * @brief Selects the active session. + * + * @param sess A pointer to the session to select. + */ + void select_session(const ipx_session *sess); + + /** + * @brief Selects the active ODID (Observation Domain ID). + * + * @param odid The ODID to select. + */ + void select_odid(uint32_t odid); + + /** + * @brief Deletes a session and its associated data. + * + * @param sess A pointer to the session to delete. + */ + void delete_session(const ipx_session *sess); + + /** + * @brief Retrieves the RecParser for a specific template. + * + * @param tmplt A pointer to the template. + * @return A reference to the corresponding RecParser. + */ + RecParser& get_parser(const fds_template *tmplt); + +private: + using TemplateMap = std::unordered_map; ///< Map of template IDs to RecParser instances. + using OdidMap = std::unordered_map; ///< Map of ODIDs to TemplateMap instances. + using SessionMap = std::unordered_map; ///< Map of sessions to OdidMap instances. + + const std::vector &m_columns; ///< Reference to the column configuration. + bool m_biflow_autoignore; ///< Whether to automatically ignore biflow records. + + TemplateMap *m_active_odid = nullptr; ///< Pointer to the active TemplateMap. + OdidMap *m_active_session = nullptr; ///< Pointer to the active OdidMap. + SessionMap m_sessions; ///< Map of sessions to their associated ODID maps. +}; From 0a8692b1086e8b4f0ea7577f719081310fd105f0 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 10 Apr 2025 17:13:08 +0200 Subject: [PATCH 13/40] Clickhouse - introduce Inserter --- .../output/clickhouse/src/inserter.cpp | 167 ++++++++++++++++++ .../output/clickhouse/src/inserter.h | 56 ++++++ 2 files changed, 223 insertions(+) create mode 100644 extra_plugins/output/clickhouse/src/inserter.cpp create mode 100644 extra_plugins/output/clickhouse/src/inserter.h diff --git a/extra_plugins/output/clickhouse/src/inserter.cpp b/extra_plugins/output/clickhouse/src/inserter.cpp new file mode 100644 index 000000000..5996a6ad1 --- /dev/null +++ b/extra_plugins/output/clickhouse/src/inserter.cpp @@ -0,0 +1,167 @@ +/** + * @file + * @author Michal Sedlak + * @brief Inserter class for inserting data into ClickHouse + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "inserter.h" +#include +#include +#include +#include + +static constexpr int ERR_TABLE_NOT_FOUND = 60; +static constexpr int STOP_TIMEOUT_SECS = 10; + +static std::vector> describe_table(clickhouse::Client &client, const std::string &table) +{ + std::vector> name_and_type; + try { + client.Select("DESCRIBE TABLE " + table, [&](const clickhouse::Block &block) { + if (block.GetColumnCount() > 0 && block.GetRowCount() > 0) { + const auto &name = block[0]->As(); + const auto &type = block[1]->As(); + for (size_t i = 0; i < block.GetRowCount(); i++) { + name_and_type.emplace_back(name->At(i), type->At(i)); + } + } + }); + } catch (const clickhouse::ServerException &exc) { + if (exc.GetCode() == ERR_TABLE_NOT_FOUND) { + throw Error("table \"{}\" does not exist", table); + } else { + throw; + } + } + return name_and_type; +} + +static void ensure_schema(clickhouse::Client &client, const std::string &table, const std::vector &columns) +{ + // Check that the database has the necessary columns + auto db_columns = describe_table(client, table); + + auto schema_hint = [&](){ + std::stringstream ss; + ss << "hint:\n"; + ss << "CREATE TABLE " << table << "(\n"; + size_t i = 0; + for (const auto& column : columns) { + const auto &clickhouse_type = type_to_clickhouse(columns[i].datatype, columns[i].nullable); + ss << " \"" << column.name << "\" " << clickhouse_type << (i < columns.size()-1 ? "," : "") << '\n'; + i++; + } + ss << ");"; + return ss.str(); + }; + + if (columns.size() != db_columns.size()) { + throw Error("config has {} columns but table \"{}\" has {}\n{}", columns.size(), table, db_columns.size(), schema_hint()); + } + + for (size_t i = 0; i < db_columns.size(); i++) { + const auto &expected_name = columns[i].name; + const auto &expected_type = type_to_clickhouse(columns[i].datatype, columns[i].nullable); + const auto &[actual_name, actual_type] = db_columns[i]; + + if (expected_name != actual_name) { + throw Error("expected column #{} in table \"{}\" to be named \"{}\" but it is \"{}\"\n{}", + i, table, expected_name, actual_name, schema_hint()); + } + + if (expected_type != actual_type) { + throw Error("expected column #{} in table \"{}\" to be of type \"{}\" but it is \"{}\"\n{}", + i, table, expected_type, actual_type, schema_hint()); + } + } +} + +Inserter::Inserter( + int id, + Logger logger, + clickhouse::ClientOptions client_opts, + std::string table_name, + const std::vector &columns, + SyncQueue &input_blocks, + SyncQueue &avail_blocks) + : m_id(id) + , m_logger(logger) + , m_client_opts(client_opts) + , m_table_name(table_name) + , m_columns(columns) + , m_input_blocks(input_blocks) + , m_avail_blocks(avail_blocks) +{} + +bool Inserter::insert(clickhouse::Block &block) +{ + bool needs_reconnect = false; + while (true) { + if (stop_requested() && secs_since_stop_requested() > STOP_TIMEOUT_SECS) { + return false; + } + + try { + if (needs_reconnect) { + m_client->ResetConnectionEndpoint(); + ensure_schema(*m_client.get(), m_table_name, m_columns); + m_logger.warning("[Worker %d] Connected to %s:%d due to error with previous endpoint", m_id, + m_client->GetCurrentEndpoint()->host.c_str(), m_client->GetCurrentEndpoint()->port); + } + m_logger.debug("[Worker %d] Inserting %d rows", m_id, block.GetRowCount()); + m_client->Insert(m_table_name, block); + break; + + } catch (const std::exception &ex) { + m_logger.error("[Worker %d] Insert failed: %s - retrying in 1 second", m_id, ex.what()); + needs_reconnect = true; + } + + if (stop_requested() && secs_since_stop_requested() > STOP_TIMEOUT_SECS) { + return false; + } + + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + + return true; +} + +void Inserter::run() +{ + m_client = std::make_unique(m_client_opts); + ensure_schema(*m_client.get(), m_table_name, m_columns); + m_logger.info("[Worker %d] Connected to %s:%d", m_id, + m_client->GetCurrentEndpoint()->host.c_str(), m_client->GetCurrentEndpoint()->port); + + while (true) { + Block *block = m_input_blocks.get(); + if (!block) { + // We might get null as a way to get unblocked and process stop signal. + // Nulls are inserted only after all the valid blocks are queued. + break; + } + + if (stop_requested() && secs_since_stop_requested() > STOP_TIMEOUT_SECS) { + break; + } + + block->block.RefreshRowCount(); + bool ok = insert(block->block); + if (!ok) { + // Do not clear the block as it could not have been inserted. + // It will be used to count the number of dropped records. + break; + } + + for (auto &column : block->columns) { + column->Clear(); + } + block->rows = 0; + m_avail_blocks.put(block); + } +} diff --git a/extra_plugins/output/clickhouse/src/inserter.h b/extra_plugins/output/clickhouse/src/inserter.h new file mode 100644 index 000000000..0c0211c5c --- /dev/null +++ b/extra_plugins/output/clickhouse/src/inserter.h @@ -0,0 +1,56 @@ +/** + * @file + * @author Michal Sedlak + * @brief Inserter class for inserting data into ClickHouse + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "worker.h" +#include "block.h" +#include "syncqueue.h" +#include "column.h" + +/** + * @class Inserter + * @brief A worker class responsible for inserting data into a ClickHouse table. + */ +class Inserter : public Worker { +public: + /** + * @brief Constructor for the Inserter class. + * @param id Unique identifier for the inserter. + * @param logger Logger instance for logging operations. + * @param client_opts Options for configuring the ClickHouse client. + * @param table_name Name of the ClickHouse table to insert data into. + * @param columns Reference to the vector of columns defining the table schema. + * @param input_blocks Reference to the queue of input blocks to be inserted. + * @param avail_blocks Reference to the queue of available blocks for reuse. + */ + Inserter( + int id, + Logger logger, + clickhouse::ClientOptions client_opts, + std::string table_name, + const std::vector &columns, + SyncQueue &input_blocks, + SyncQueue &avail_blocks); + +private: + int m_id; ///< Unique identifier for the inserter. + Logger m_logger; ///< Logger instance for logging operations. + clickhouse::ClientOptions m_client_opts; ///< Options for configuring the ClickHouse client. + std::string m_table_name; ///< Name of the ClickHouse table to insert data into. + const std::vector &m_columns; ///< Reference to the vector of columns defining the table schema. + SyncQueue &m_input_blocks; ///< Reference to the queue of input blocks to be inserted. + SyncQueue &m_avail_blocks; ///< Reference to the queue of available blocks for reuse. + std::unique_ptr m_client; ///< Pointer to the ClickHouse client instance. + + void run() override; + + bool insert(clickhouse::Block &block); +}; From 9731b74deb14b67b7c7c73f2f51d4ba53e456cdc Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 14 Nov 2024 12:26:17 +0100 Subject: [PATCH 14/40] Clickhouse - add main plugin class and plugin interface implementation --- extra_plugins/output/clickhouse/src/main.cpp | 80 +++++ .../output/clickhouse/src/plugin.cpp | 280 ++++++++++++++++++ extra_plugins/output/clickhouse/src/plugin.h | 86 ++++++ 3 files changed, 446 insertions(+) create mode 100644 extra_plugins/output/clickhouse/src/main.cpp create mode 100644 extra_plugins/output/clickhouse/src/plugin.cpp create mode 100644 extra_plugins/output/clickhouse/src/plugin.h diff --git a/extra_plugins/output/clickhouse/src/main.cpp b/extra_plugins/output/clickhouse/src/main.cpp new file mode 100644 index 000000000..7314b259a --- /dev/null +++ b/extra_plugins/output/clickhouse/src/main.cpp @@ -0,0 +1,80 @@ +/** + * @file + * @author Michal Sedlak + * @brief Plugin interface implementation + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "plugin.h" + +#include + +#include + +/** Plugin description */ +IPX_API struct ipx_plugin_info ipx_plugin_info = { + // Plugin identification name + "clickhouse", + // Brief description of plugin + "Output plugin that stores flow records to ClickHouse database.", + // Plugin type + IPX_PT_OUTPUT, + // Configuration flags (reserved for future use) + 0, + // Plugin version string (like "1.2.3") + "1.0.0", + // Minimal IPFIXcol version string (like "1.2.3") + "2.8.0" +}; + +int +ipx_plugin_init(ipx_ctx_t *ctx, const char *xml_config) +{ + std::unique_ptr plugin; + try { + plugin = std::make_unique(ctx, xml_config); + } catch (const std::exception &ex) { + IPX_CTX_ERROR(ctx, "An unexpected exception has occured: %s", ex.what()); + return IPX_ERR_DENIED; + } catch (...) { + IPX_CTX_ERROR(ctx, "An unexpected exception has occured."); + return IPX_ERR_DENIED; + } + ipx_ctx_private_set(ctx, plugin.release()); + return IPX_OK; +} + +void +ipx_plugin_destroy(ipx_ctx_t *ctx, void *priv) +{ + (void) ctx; + Plugin *plugin = reinterpret_cast(priv); + try { + plugin->stop(); + } catch (const std::exception &ex) { + IPX_CTX_ERROR(ctx, "An unexpected exception has occured: %s", ex.what()); + } catch (...) { + IPX_CTX_ERROR(ctx, "An unexpected exception has occured."); + } + delete plugin; +} + +int +ipx_plugin_process(ipx_ctx_t *ctx, void *priv, ipx_msg_t *msg) +{ + Plugin *plugin = reinterpret_cast(priv); + try { + plugin->process(msg); + } catch (const std::exception &ex) { + IPX_CTX_ERROR(ctx, "An unexpected exception has occured: %s", ex.what()); + return IPX_ERR_DENIED; + } catch (...) { + IPX_CTX_ERROR(ctx, "An unexpected exception has occured."); + return IPX_ERR_DENIED; + } + + return IPX_OK; +} diff --git a/extra_plugins/output/clickhouse/src/plugin.cpp b/extra_plugins/output/clickhouse/src/plugin.cpp new file mode 100644 index 000000000..61d9acd5e --- /dev/null +++ b/extra_plugins/output/clickhouse/src/plugin.cpp @@ -0,0 +1,280 @@ +/** + * @file + * @author Michal Sedlak + * @brief Main plugin class + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "datatype.h" +#include "plugin.h" + +#include +#include +#include + +static std::vector prepare_columns(std::vector &columns_cfg) +{ + std::vector columns; + + for (const auto &column_cfg : columns_cfg) { + Column column{}; + DataType type; + + if (std::holds_alternative(column_cfg.source)) { + const fds_iemgr_elem *elem = std::get(column_cfg.source); + type = type_from_ipfix(elem->data_type); + column.elem = elem; + + } else if (std::holds_alternative(column_cfg.source)) { + const fds_iemgr_alias *alias = std::get(column_cfg.source); + type = find_common_type(*alias); + column.alias = alias; + + } else /* if (std::holds_alternative(column_cfg.source)) */ { + type = DataType::UInt32; + column.special = std::get(column_cfg.source); + + } + + column.name = column_cfg.name; + column.datatype = type; + column.nullable = column_cfg.nullable; + + columns.emplace_back(std::move(column)); + } + + return columns; +} + + +Plugin::Plugin(ipx_ctx_t *ctx, const char *xml_config) + : m_logger(ctx) + , m_stats(m_logger, *this) +{ + // Subscribe to periodic messages aswell to ensure data export even when no data is coming + ipx_msg_mask_t new_mask = IPX_MSG_IPFIX | IPX_MSG_PERIODIC | IPX_MSG_SESSION; + int rc = ipx_ctx_subscribe(ctx, &new_mask, nullptr); + if (rc != IPX_OK) { + throw Error("ipx_ctx_subscribe() failed with error code {}", rc); + } + + // Parse config + m_config = parse_config(xml_config, ipx_ctx_iemgr_get(ctx)); + + std::vector endpoints; + for (const Config::Endpoint &endpoint_cfg : m_config.connection.endpoints) { + endpoints.push_back(clickhouse::Endpoint{endpoint_cfg.host, endpoint_cfg.port}); + } + + m_columns = prepare_columns(m_config.columns); + m_rec_parsers = std::make_unique(m_columns, m_config.biflow_empty_autoignore); + + // Prepare blocks + for (unsigned int i = 0; i < m_config.blocks; i++) { + std::unique_ptr blk = std::make_unique(); + for (const auto &column : m_columns) { + blk->columns.emplace_back(make_column(column.datatype, column.nullable)); + blk->block.AppendColumn(column.name, blk->columns.back()); + } + m_blocks.emplace_back(std::move(blk)); + m_avail_blocks.put(m_blocks.back().get()); + } + + // Prepare inserters + for (unsigned int i = 0; i < m_config.inserter_threads; i++) { + clickhouse::ClientOptions client_opts = clickhouse::ClientOptions() + .SetEndpoints(endpoints) + .SetUser(m_config.connection.user) + .SetPassword(m_config.connection.password) + .SetDefaultDatabase(m_config.connection.database); + std::unique_ptr ins = std::make_unique( + i, + m_logger, + client_opts, + m_config.connection.table, + m_columns, + m_filled_blocks, + m_avail_blocks); + + m_inserters.emplace_back(std::move(ins)); + } + + m_logger.info("Starting inserters"); + for (auto &ins : m_inserters) { + ins->start(); + } + + + m_logger.info("ClickHouse plugin is ready"); +} + +void Plugin::extract_values(ipx_msg_ipfix_t *msg, RecParser &parser, Block &block, bool rev) +{ + std::size_t n_columns = m_columns.size(); + bool has_value; + ValueVariant value; + + for (std::size_t i = 0; i < n_columns; i++) { + has_value = false; + + if (m_columns[i].special == SpecialField::ODID) { + value = ipx_msg_ipfix_get_ctx(msg)->odid; + has_value = true; + + } else { + fds_drec_field &field = parser.get_column(i, rev); + if (field.data != nullptr) { + try { + value = get_value(m_columns[i].datatype, field); + has_value = true; + } catch (const ConversionError& err) { + m_logger.error("Field conversion failed (field #%d, \"%s\"): %s", i, m_columns[i].name, err.what()); + } + } + } + + write_to_column(m_columns[i].datatype, m_columns[i].nullable, *block.columns[i].get(), has_value ? &value : nullptr); + } + block.rows++; +} + +int +Plugin::process_record(ipx_msg_ipfix_t *msg, fds_drec &rec, Block &block) +{ + int ret = 0; + RecParser &parser = m_rec_parsers->get_parser(rec.tmplt); + parser.parse_record(rec); + + if (!parser.skip_fwd()) { + extract_values(msg, parser, block, false); + ret++; + } + + if (!parser.skip_rev()) { + extract_values(msg, parser, block, true); + ret++; + } + + return ret; +} + +void +Plugin::process_session_msg(ipx_msg_session_t *msg) +{ + if (ipx_msg_session_get_event(msg) == IPX_MSG_SESSION_CLOSE) { + const ipx_session *sess = ipx_msg_session_get_session(msg); + m_rec_parsers->delete_session(sess); + } +} + +void +Plugin::process_ipfix_msg(ipx_msg_ipfix_t *msg) +{ + // get new block if we don't have one + if (m_current_block == nullptr) { + if (m_config.nonblocking) { + std::optional maybe_block = m_avail_blocks.try_get(); + if (maybe_block.has_value()) { + m_current_block = maybe_block.value(); + } else { + // no available blocks and we are in a non-blocking mode, drop the message + uint32_t drec_cnt = ipx_msg_ipfix_get_drec_cnt(msg); + m_stats.add_dropped(drec_cnt); + return; + } + } else { + m_current_block = m_avail_blocks.get(); + } + } + + // setup rec parser + const ipx_msg_ctx *msg_ctx = ipx_msg_ipfix_get_ctx(msg); + if (msg_ctx->session->type == FDS_SESSION_SCTP) { + throw std::runtime_error("SCTP is not supported at this time"); + } + m_rec_parsers->select_session(msg_ctx->session); + m_rec_parsers->select_odid(msg_ctx->odid); + + // go through all the records + uint32_t drec_cnt = ipx_msg_ipfix_get_drec_cnt(msg); + uint32_t rows_count = 0; + for (uint32_t idx = 0; idx < drec_cnt; idx++) { + ipx_ipfix_record *rec = ipx_msg_ipfix_get_drec(msg, idx); + uint32_t rows_inserted = process_record(msg, rec->rec, *m_current_block); + rows_count += rows_inserted; + } + + m_stats.add_recs(drec_cnt); + m_stats.add_rows(rows_count); +} + +void +Plugin::process(ipx_msg_t *msg) +{ + if (ipx_msg_get_type(msg) == IPX_MSG_SESSION) { + process_session_msg(ipx_msg_base2session(msg)); + + } else if (ipx_msg_get_type(msg) == IPX_MSG_IPFIX) { + ipx_msg_ipfix_t *ipfix_msg = ipx_msg_base2ipfix(msg); + process_ipfix_msg(ipfix_msg); + } + + time_t now = std::time(nullptr); + + // Send the block for insertion if it is sufficiently full or a block hasn't been sent in a long enough time + if (m_current_block) { + bool nonempty = m_current_block->rows > 0; + bool thresh_reached = m_current_block->rows >= m_config.block_insert_threshold; + bool timeout_reached = uint64_t(now - m_last_insert_time) >= m_config.block_insert_max_delay_secs; + + if (nonempty && (thresh_reached || timeout_reached)) { + m_filled_blocks.put(m_current_block); + m_current_block = nullptr; + m_last_insert_time = now; + } + } + + // Print stats + m_stats.print_stats_throttled(now); + + // Check for any exceptions thrown by workers + for (auto &ins : m_inserters) { + ins->check_error(); + } + +} + +void Plugin::stop() +{ + // Export what's left in the last block + if (m_current_block && m_current_block->rows > 0) { + m_filled_blocks.put(m_current_block); + m_current_block = nullptr; + } + + // Stop all the threads and wait for them to finish + m_logger.info("Sending stop signal to inserter threads..."); + for (auto &ins : m_inserters) { + ins->request_stop(); + } + for (const auto &ins : m_inserters) { + (void) ins; + // Wake up the inserter threads in case they are waiting on a .get() + m_filled_blocks.put(nullptr); + } + + m_logger.info("Waiting for inserter threads to finish..."); + for (auto &ins : m_inserters) { + ins->join(); + } + + std::size_t drop_count = 0; + for (const auto& block : m_blocks) { + drop_count += block->rows; + } + m_logger.warning("%zu rows could not have been inserted and have been dropped due to termination timeout", + drop_count); +} diff --git a/extra_plugins/output/clickhouse/src/plugin.h b/extra_plugins/output/clickhouse/src/plugin.h new file mode 100644 index 000000000..d2c80f7a6 --- /dev/null +++ b/extra_plugins/output/clickhouse/src/plugin.h @@ -0,0 +1,86 @@ +/** + * @file + * @author Michal Sedlak + * @brief Main plugin class + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "block.h" +#include "clickhouse.h" +#include "column.h" +#include "common.h" +#include "config.h" +#include "inserter.h" +#include "recparser.h" +#include "stats.h" + +#include +#include +#include + + +/** + * @class Plugin + * @brief Class representing the ClickHouse output plugin + */ +class Plugin : Nonmoveable, Noncopyable { +public: + /** + * @brief Instantiate the plugin instance + * + * @param ctx The collector context for logging + * @param xml_config The config in form of a XML string + */ + Plugin(ipx_ctx_t *ctx, const char *xml_config); + + /** + * @brief Process a collector message + * + * @param msg The collector message + */ + void process(ipx_msg_t *msg); + + /** + * @brief Stop the plugin and wait till it is stopped (blocking) + */ + void stop(); + +private: + friend class Stats; + + Logger m_logger; + Config m_config; + std::vector m_columns; + + std::vector> m_blocks; + + std::vector> m_inserters; + + SyncQueue m_avail_blocks; + SyncQueue m_filled_blocks; + + Block *m_current_block = nullptr; + + Stats m_stats; + + std::time_t m_last_insert_time = 0; + + std::unique_ptr m_rec_parsers; + + void + process_ipfix_msg(ipx_msg_ipfix_t *msg); + + void + process_session_msg(ipx_msg_session_t *msg); + + int + process_record(ipx_msg_ipfix_t *msg, fds_drec &rec, Block &block); + + void + extract_values(ipx_msg_ipfix_t *msg, RecParser &parser, Block &block, bool rev); +}; From 9c6c5c66f0d1ef43457ec80e0925bb45dcb1023a Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 14 Nov 2024 12:23:51 +0100 Subject: [PATCH 15/40] Clickhouse - add build files --- .../output/clickhouse/CMakeLists.txt | 90 +++++++++++++++++++ .../CMakeModules/FindIPFIXcol2.cmake | 39 ++++++++ .../clickhouse/CMakeModules/FindRst2Man.cmake | 30 +++++++ .../CMakeModules/clickhouse-cpp.cmake | 21 +++++ .../output/clickhouse/CMakeModules/fmt.cmake | 20 +++++ .../CMakeModules/install_dirs.cmake | 43 +++++++++ 6 files changed, 243 insertions(+) create mode 100644 extra_plugins/output/clickhouse/CMakeLists.txt create mode 100644 extra_plugins/output/clickhouse/CMakeModules/FindIPFIXcol2.cmake create mode 100644 extra_plugins/output/clickhouse/CMakeModules/FindRst2Man.cmake create mode 100644 extra_plugins/output/clickhouse/CMakeModules/clickhouse-cpp.cmake create mode 100644 extra_plugins/output/clickhouse/CMakeModules/fmt.cmake create mode 100644 extra_plugins/output/clickhouse/CMakeModules/install_dirs.cmake diff --git a/extra_plugins/output/clickhouse/CMakeLists.txt b/extra_plugins/output/clickhouse/CMakeLists.txt new file mode 100644 index 000000000..5af677c35 --- /dev/null +++ b/extra_plugins/output/clickhouse/CMakeLists.txt @@ -0,0 +1,90 @@ +cmake_minimum_required(VERSION 3.12) +project(ipfixcol2-clichouse-output) + +# Description of the project +set(CH_DESCRIPTION + "Output plugin for IPFIXcol2 that store flow records to ClickHouse database." +) + +set(CH_VERSION_MAJOR 1) +set(CH_VERSION_MINOR 0) +set(CH_VERSION_PATCH 0) +set(CH_VERSION + ${CH_VERSION_MAJOR}.${CH_VERSION_MINOR}.${CH_VERSION_PATCH}) + +include(CMakeModules/install_dirs.cmake) +include(CMakeModules/clickhouse-cpp.cmake) +include(CMakeModules/fmt.cmake) +include(CheckCCompilerFlag) +include(CheckCXXCompilerFlag) +# Include custom FindXXX modules +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules") + +# Find IPFIXcol +find_package(IPFIXcol2 2.3.0 REQUIRED) + +# Set default build type if not specified by user +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Debug + CACHE STRING "Choose type of build (Release/Debug)." FORCE) +endif() + +option(ENABLE_DOC_MANPAGE "Enable manual page building" ON) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS ON) + +# Hard coded definitions +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden") +set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG") +set(CMAKE_C_FLAGS_DEBUG "-g -O0 -Wall -Wextra -pedantic") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") +set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG") +set(CMAKE_CXX_FLAGS_DEBUG "-g -O0 -Wall -Wextra -pedantic") + +# Header files for source code building +include_directories( + "${IPFIXCOL2_INCLUDE_DIRS}" # IPFIXcol2 header files +) + +# Create a linkable module +add_library(clickhouse-output MODULE + src/common.cpp + src/config.cpp + src/datatype.cpp + src/main.cpp + src/plugin.cpp + src/inserter.cpp + src/recparser.cpp + src/stats.cpp +) + +target_link_libraries(clickhouse-output PRIVATE clickhouse::client fmt::fmt) + +install( + TARGETS clickhouse-output + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" +) + +if (ENABLE_DOC_MANPAGE) + find_package(Rst2Man) + if (NOT RST2MAN_FOUND) + message(FATAL_ERROR "rst2man is not available. Install python-docutils or disable manual page generation (-DENABLE_DOC_MANPAGE=False)") + endif() + + # Build a manual page + set(SRC_FILE "${CMAKE_CURRENT_SOURCE_DIR}/doc/ipfixcol2-clickhouse-output.7.rst") + set(DST_FILE "${CMAKE_CURRENT_BINARY_DIR}/ipfixcol2-clickhouse-output.7") + + add_custom_command(TARGET clickhouse-output PRE_BUILD + COMMAND ${RST2MAN_EXECUTABLE} --syntax-highlight=none ${SRC_FILE} ${DST_FILE} + DEPENDS ${SRC_FILE} + VERBATIM + ) + + install( + FILES "${DST_FILE}" + DESTINATION "${INSTALL_DIR_MAN}/man7" + ) +endif() diff --git a/extra_plugins/output/clickhouse/CMakeModules/FindIPFIXcol2.cmake b/extra_plugins/output/clickhouse/CMakeModules/FindIPFIXcol2.cmake new file mode 100644 index 000000000..c42e0d7cf --- /dev/null +++ b/extra_plugins/output/clickhouse/CMakeModules/FindIPFIXcol2.cmake @@ -0,0 +1,39 @@ +# IPFIXCOL2_FOUND - System has IPFIXcol +# IPFIXCOL2_INCLUDE_DIRS - The IPFIXcol include directories +# IPFIXCOL2_DEFINITIONS - Compiler switches required for using IPFIXcol + +# use pkg-config to get the directories and then use these values +# in the find_path() and find_library() calls +find_package(PkgConfig) +pkg_check_modules(PC_IPFIXCOL QUIET ipfixcol2) +set(IPFIXCOL2_DEFINITIONS ${PC_IPFIXCOL_CFLAGS_OTHER}) + +find_path( + IPFIXCOL2_INCLUDE_DIR ipfixcol2.h + HINTS ${PC_IPFIXCOL_INCLUDEDIR} ${PC_IPFIXCOL_INCLUDE_DIRS} + PATH_SUFFIXES include +) + +if (PC_IPFIXCOL_VERSION) + # Version extracted from pkg-config + set(IPFIXCOL_VERSION_STRING ${PC_IPFIXCOL_VERSION}) +elseif(IPFIXCOL2_INCLUDE_DIR AND EXISTS "${IPFIXCOL2_INCLUDE_DIR}/ipfixcol2/api.h") + # Try to extract library version from a header file + file(STRINGS "${IPFIXCOL2_INCLUDE_DIR}/ipfixcol2/api.h" ipfixcol_version_str + REGEX "^#define[\t ]+IPX_API_VERSION_STR[\t ]+\".*\"") + + string(REGEX REPLACE "^#define[\t ]+IPX_API_VERSION_STR[\t ]+\"([^\"]*)\".*" "\\1" + IPFIXCOL_VERSION_STRING "${ipfixcol_version_str}") + unset(ipfixcol_version_str) +endif() + +# handle the QUIETLY and REQUIRED arguments and set IPFIXCOL2_FOUND to TRUE +# if all listed variables are TRUE +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(IPFIXcol2 + REQUIRED_VARS IPFIXCOL2_INCLUDE_DIR + VERSION_VAR IPFIXCOL_VERSION_STRING +) + +set(IPFIXCOL2_INCLUDE_DIRS ${IPFIXCOL2_INCLUDE_DIR}) +mark_as_advanced(IPFIXCOL2_INCLUDE_DIR) diff --git a/extra_plugins/output/clickhouse/CMakeModules/FindRst2Man.cmake b/extra_plugins/output/clickhouse/CMakeModules/FindRst2Man.cmake new file mode 100644 index 000000000..df213bce1 --- /dev/null +++ b/extra_plugins/output/clickhouse/CMakeModules/FindRst2Man.cmake @@ -0,0 +1,30 @@ +# RST2MAN_FOUND - true if the program was found +# RST2MAN_VERSION - version of rst2man +# RST2MAN_EXECUTABLE - path to the rst2man program + +find_program(RST2MAN_EXECUTABLE + NAMES rst2man rst2man.py rst2man-3 rst2man-3.py + DOC "The Python Docutils generator of Unix Manpages from reStructuredText" +) + +if (RST2MAN_EXECUTABLE) + # Get the version string + execute_process( + COMMAND ${RST2MAN_EXECUTABLE} --version + OUTPUT_VARIABLE rst2man_version_str + ) + # Expected format: rst2man (Docutils 0.13.1 [release], Python 2.7.15, on linux2) + string(REGEX REPLACE "^rst2man[\t ]+\\(Docutils[\t ]+([^\t ]*).*" "\\1" + RST2MAN_VERSION "${rst2man_version_str}") + unset(rst2man_version_str) +endif() + +# handle the QUIETLY and REQUIRED arguments and set RST2MAN_FOUND to TRUE +# if all listed variables are set +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Rst2Man + REQUIRED_VARS RST2MAN_EXECUTABLE + VERSION_VAR RST2MAN_VERSION +) + +mark_as_advanced(RST2MAN_EXECUTABLE RST2MAN_VERSION) diff --git a/extra_plugins/output/clickhouse/CMakeModules/clickhouse-cpp.cmake b/extra_plugins/output/clickhouse/CMakeModules/clickhouse-cpp.cmake new file mode 100644 index 000000000..62827ecb5 --- /dev/null +++ b/extra_plugins/output/clickhouse/CMakeModules/clickhouse-cpp.cmake @@ -0,0 +1,21 @@ +# clickhouse-cpp library (C++ client for ClickHouse) +# +# The project consists of a library that can be independently +# added as dependency: +# - clickhouse::client + +include(FetchContent) +set(FETCHCONTENT_QUIET OFF) + +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(BUILD_SHARED_LIBS OFF) +FetchContent_Declare( + clickhouse + GIT_REPOSITORY "https://github.com/ClickHouse/clickhouse-cpp.git" + GIT_TAG "v2.5.1" + GIT_SHALLOW ON +) + +FetchContent_MakeAvailable(clickhouse) +add_library(clickhouse::client ALIAS clickhouse-cpp-lib) +set(BUILD_SHARED_LIBS ON) diff --git a/extra_plugins/output/clickhouse/CMakeModules/fmt.cmake b/extra_plugins/output/clickhouse/CMakeModules/fmt.cmake new file mode 100644 index 000000000..1176a5c2f --- /dev/null +++ b/extra_plugins/output/clickhouse/CMakeModules/fmt.cmake @@ -0,0 +1,20 @@ +# fmt library +# +# The project consists of a library that can be independently +# added as dependency: +# - fmt::fmt + +include(FetchContent) +set(FETCHCONTENT_QUIET OFF) + +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(BUILD_SHARED_LIBS OFF) +FetchContent_Declare( + fmt + GIT_REPOSITORY "https://github.com/fmtlib/fmt" + GIT_TAG "11.0.2" + GIT_SHALLOW ON +) + +FetchContent_MakeAvailable(fmt) +set(BUILD_SHARED_LIBS ON) diff --git a/extra_plugins/output/clickhouse/CMakeModules/install_dirs.cmake b/extra_plugins/output/clickhouse/CMakeModules/install_dirs.cmake new file mode 100644 index 000000000..6b540e084 --- /dev/null +++ b/extra_plugins/output/clickhouse/CMakeModules/install_dirs.cmake @@ -0,0 +1,43 @@ +# The purpose of this file is to automatically determine install directories +# +# If no directories are defined, use GNU install directories by default. +# However, in case of RPM build, install directories are typically passed +# to CMake as definitions that overwrites the default paths. +# + +include(GNUInstallDirs) + +# Binary directories +set(INSTALL_DIR_BIN ${CMAKE_INSTALL_FULL_BINDIR}) + +# Library directories +if (DEFINED LIB_INSTALL_DIR) + set(INSTALL_DIR_LIB ${LIB_INSTALL_DIR}) +else() + set(INSTALL_DIR_LIB ${CMAKE_INSTALL_FULL_LIBDIR}) +endif() + +# Include directories +if (DEFINED INCLUDE_INSTALL_DIR) + set(INSTALL_DIR_INCLUDE ${INCLUDE_INSTALL_DIR}) +else() + set(INSTALL_DIR_INCLUDE ${CMAKE_INSTALL_FULL_INCLUDEDIR}) +endif() + +# System configuration +if (DEFINED SYSCONF_INSTALL_DIR) + set(INSTALL_DIR_SYSCONF ${SYSCONF_INSTALL_DIR}) +else() + set(INSTALL_DIR_SYSCONF ${CMAKE_INSTALL_FULL_SYSCONFDIR}) +endif() + +# Share files (architecture independend data) +if (DEFINED SHARE_INSTALL_PREFIX) + set(INSTALL_DIR_SHARE ${SHARE_INSTALL_PREFIX}) +else() + set(INSTALL_DIR_SHARE ${CMAKE_INSTALL_FULL_DATAROOTDIR}) +endif() + +set(INSTALL_DIR_INFO "${INSTALL_DIR_SHARE}/info/") +set(INSTALL_DIR_MAN "${INSTALL_DIR_SHARE}/man/") +set(INSTALL_DIR_DOC "${INSTALL_DIR_SHARE}/doc/${CMAKE_PROJECT_NAME}/") From fcc1cc837693ae7c0ce2d8a9ab752da6ad4be6b4 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Wed, 26 Mar 2025 13:24:15 +0100 Subject: [PATCH 16/40] Clickhouse - add spec file --- .../ipfixcol2-clickhouse-output.spec | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 extra_plugins/output/clickhouse/ipfixcol2-clickhouse-output.spec diff --git a/extra_plugins/output/clickhouse/ipfixcol2-clickhouse-output.spec b/extra_plugins/output/clickhouse/ipfixcol2-clickhouse-output.spec new file mode 100644 index 000000000..6f0e5b74f --- /dev/null +++ b/extra_plugins/output/clickhouse/ipfixcol2-clickhouse-output.spec @@ -0,0 +1,51 @@ +Name: ipfixcol2-clickhouse-output +Version: 1.0.0 +Release: 1%{?dist} +Summary: Plugin for export of IPFIX flow records into a ClickHouse database. + +License: BSD +URL: http://www.liberouter.org/ +Source0: %{name}-%{version}.tar.gz +Group: Liberouter +Vendor: CESNET, z.s.p.o. +Packager: + +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release} +BuildRequires: gcc >= 7.0, gcc-c++ >= 7.0, cmake >= 2.8.8, make +BuildRequires: ipfixcol2-devel, libfds-devel, /usr/bin/rst2man +Requires: ipfixcol2 >= 2.0.0, libfds >= 0.1.0 + +%description +Plugin for converting IPFIX data to UniRec format. + +%prep +%autosetup + +#%post +#/sbin/ldconfig + +#%postun +#/sbin/ldconfig + +# Build source code +%build +%global _vpath_builddir . +%cmake . +make %{?_smp_mflags} + +# Perform installation into build directory +%install +mkdir -p %{buildroot}/install +make install DESTDIR=%{buildroot}/install +mkdir -p %{buildroot}/%{_libdir}/ipfixcol2/ %{buildroot}/%{_mandir}/man7/ +mv %{buildroot}/install/%{_libdir}/ipfixcol2/*.so* %{buildroot}/%{_libdir}/ipfixcol2/ +mv %{buildroot}/install/%{_mandir}/man7/*.7* %{buildroot}/%{_mandir}/man7/ +rm -rf %{buildroot}/install + +%files +%{_libdir}/ipfixcol2/*.so* +%{_mandir}/man7/*.7* + +%changelog +* Thu Jan 23 2025 Michal Sedlak 1.0.0-1 +- Initial RPM release From 0acc3a52695f21c703376668e08df35812076bdc Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Tue, 6 May 2025 16:18:45 +0200 Subject: [PATCH 17/40] Overview - introduce plugin --- README.rst | 2 + src/plugins/output/CMakeLists.txt | 1 + src/plugins/output/overview/CMakeLists.txt | 28 +++++ src/plugins/output/overview/README.rst | 70 +++++++++++ .../doc/ipfixcol2-overview-output.7.rst | 20 +++ src/plugins/output/overview/src/config.cpp | 66 ++++++++++ src/plugins/output/overview/src/config.h | 30 +++++ src/plugins/output/overview/src/main.cpp | 78 ++++++++++++ src/plugins/output/overview/src/plugin.cpp | 119 ++++++++++++++++++ src/plugins/output/overview/src/plugin.h | 52 ++++++++ 10 files changed, 466 insertions(+) create mode 100644 src/plugins/output/overview/CMakeLists.txt create mode 100644 src/plugins/output/overview/README.rst create mode 100644 src/plugins/output/overview/doc/ipfixcol2-overview-output.7.rst create mode 100644 src/plugins/output/overview/src/config.cpp create mode 100644 src/plugins/output/overview/src/config.h create mode 100644 src/plugins/output/overview/src/main.cpp create mode 100644 src/plugins/output/overview/src/plugin.cpp create mode 100644 src/plugins/output/overview/src/plugin.h diff --git a/README.rst b/README.rst index b1c14d8d2..6e2b3ca33 100644 --- a/README.rst +++ b/README.rst @@ -60,6 +60,8 @@ network interface and a port. Multiple instances of these plugins can run concur it on standard output - `Time Check `_ - flow timestamp check - `Dummy `_ - simple output module example +- `Overview `_ - get a quick overview of the IPFIX + fields the collector is receiving from the probe - `lnfstore `_ (*) - store all flows in nfdump compatible format for long-term preservation - `UniRec `_ (*) - send flow records in UniRec format diff --git a/src/plugins/output/CMakeLists.txt b/src/plugins/output/CMakeLists.txt index 11681bdd4..ef0c76985 100644 --- a/src/plugins/output/CMakeLists.txt +++ b/src/plugins/output/CMakeLists.txt @@ -7,3 +7,4 @@ add_subdirectory(timecheck) add_subdirectory(viewer) add_subdirectory(ipfix) add_subdirectory(forwarder) +add_subdirectory(overview) diff --git a/src/plugins/output/overview/CMakeLists.txt b/src/plugins/output/overview/CMakeLists.txt new file mode 100644 index 000000000..84502fbb0 --- /dev/null +++ b/src/plugins/output/overview/CMakeLists.txt @@ -0,0 +1,28 @@ +# Create a linkable module +add_library(overview-output MODULE + src/main.cpp + src/plugin.cpp + src/config.cpp +) + +install( + TARGETS overview-output + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" +) + +if (ENABLE_DOC_MANPAGE) + # Build a manual page + set(SRC_FILE "${CMAKE_CURRENT_SOURCE_DIR}/doc/ipfixcol2-overview-output.7.rst") + set(DST_FILE "${CMAKE_CURRENT_BINARY_DIR}/ipfixcol2-overview-output.7") + + add_custom_command(TARGET overview-output PRE_BUILD + COMMAND ${RST2MAN_EXECUTABLE} --syntax-highlight=none ${SRC_FILE} ${DST_FILE} + DEPENDS ${SRC_FILE} + VERBATIM + ) + + install( + FILES "${DST_FILE}" + DESTINATION "${INSTALL_DIR_MAN}/man7" + ) +endif() diff --git a/src/plugins/output/overview/README.rst b/src/plugins/output/overview/README.rst new file mode 100644 index 000000000..a6ce7a0b5 --- /dev/null +++ b/src/plugins/output/overview/README.rst @@ -0,0 +1,70 @@ +Overview output plugin +====================== + +Provides an overview of elements that the collector is receiving. + +This can be useful if you need to quickly figure out the IPFIX fields your flow +probe is sending to the collector. + +The overview is printed to stdout upon exit as a JSON. + +Example output +-------------- + +.. code-block:: json + + { + "total_number_of_records": 24985802, + "elements": [ + { + "pen": 0, + "id": 136, + "data_type": "unsigned8", + "name": "iana:flowEndReason", + "aliases": [ ], + "in_number_of_records": 1112239, + "in_percent_of_records": 4.45 + }, + { + "pen": 0, + "id": 1, + "data_type": "unsigned64", + "name": "iana:octetDeltaCount", + "aliases": [ "bytes" ], + "in_number_of_records": 1112239, + "in_percent_of_records": 4.45 + }, + ... + ] + } + + +If a certain field only has a `pen` and `id` set with certain fields such as +`name` being `null`, it means that the collector is missing a definition of the +field. Field definitions are defined as XML files and can be by default found +in `/etc/libfds/` if ipfixcol2 is installed system-wide. + +Definitions that come with ipfixcol2 are stored in `/etc/libfds/system` +directory. Your own definitions can be supplied in the `/etc/libfds/user` +directory. See one of the files in `/etc/libfds/system` for the structure of +the XML files. + + +Example configuration +--------------------- + +.. code-block:: xml + + + Overview output + overview + + false + + + +Parameters +---------- + +:``skipOptionsTemplates``: + Options templates and options template records are skipped. [default: false] diff --git a/src/plugins/output/overview/doc/ipfixcol2-overview-output.7.rst b/src/plugins/output/overview/doc/ipfixcol2-overview-output.7.rst new file mode 100644 index 000000000..20c43efe2 --- /dev/null +++ b/src/plugins/output/overview/doc/ipfixcol2-overview-output.7.rst @@ -0,0 +1,20 @@ +============================= + ipfixcol2-overview-output +============================= + +-------------------------- +Overview (output plugin) +-------------------------- + +:Author: Michal Sedlak (sedlakm@cesnet.cz) +:Date: 2025-29-04 +:Copyright: Copyright © 2025 CESNET, z.s.p.o. +:Version: 0.1 +:Manual section: 7 +:Manual group: IPFIXcol collector + +Description +----------- + +.. include:: ../README.rst + :start-line: 3 diff --git a/src/plugins/output/overview/src/config.cpp b/src/plugins/output/overview/src/config.cpp new file mode 100644 index 000000000..7a7dd9734 --- /dev/null +++ b/src/plugins/output/overview/src/config.cpp @@ -0,0 +1,66 @@ +/** + * @file + * @author Michal Sedlak + * @brief Configuration parsing and representation + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "config.h" + +#include + +#include +#include + +namespace args { + +enum { + SKIP_OPTIONS_TEMPLATES +}; + +static const struct fds_xml_args root[] = { + FDS_OPTS_ROOT ("params"), + FDS_OPTS_ELEM (SKIP_OPTIONS_TEMPLATES, "skipOptionsTemplates", FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT), + FDS_OPTS_END, +}; + +} + +static void parse_root(fds_xml_ctx_t *root_ctx, Config &config) +{ + const fds_xml_cont *content; + + while (fds_xml_next(root_ctx, &content) == FDS_OK) { + if (content->id == args::SKIP_OPTIONS_TEMPLATES) { + config.skip_option_templates = content->val_bool; + } + } +} + +Config parse_config(const char *xml_string) +{ + Config config{}; + + std::unique_ptr parser(fds_xml_create(), &fds_xml_destroy); + if (!parser) { + throw std::runtime_error("Failed to create an XML parser!"); + } + + if (fds_xml_set_args(parser.get(), args::root) != FDS_OK) { + std::string err = fds_xml_last_err(parser.get()); + throw std::runtime_error("Failed to parse the description of an XML document: " + err); + } + + fds_xml_ctx_t *root_ctx = fds_xml_parse_mem(parser.get(), xml_string, true); + if (!root_ctx) { + std::string err = fds_xml_last_err(parser.get()); + throw std::runtime_error("Failed to parse the configuration: " + err); + } + + parse_root(root_ctx, config); + + return config; +} diff --git a/src/plugins/output/overview/src/config.h b/src/plugins/output/overview/src/config.h new file mode 100644 index 000000000..1fcefb260 --- /dev/null +++ b/src/plugins/output/overview/src/config.h @@ -0,0 +1,30 @@ +/** + * @file + * @author Michal Sedlak + * @brief Configuration parsing and representation + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include + +/** + * @class Config + * @brief A struct containing all the configurable plugin parameters + */ +struct Config { + bool skip_option_templates = false; +}; + +/** + * @brief Parse a XML config into a structured form + * + * @param xml The config as a XML string + * @param iemgr The iemgr instance + * @return The parsed config + */ +Config parse_config(const char *xml); diff --git a/src/plugins/output/overview/src/main.cpp b/src/plugins/output/overview/src/main.cpp new file mode 100644 index 000000000..e1b006aeb --- /dev/null +++ b/src/plugins/output/overview/src/main.cpp @@ -0,0 +1,78 @@ +/** + * @file + * @author Michal Sedlak + * @brief Main plugin entrypoint + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include "plugin.h" + +/** Plugin description */ +IPX_API struct ipx_plugin_info ipx_plugin_info = { + // Plugin identification name + .name = "overview", + // Brief description of plugin + .dsc = "Overview output plugin.", + // Plugin type + .type = IPX_PT_OUTPUT, + // Configuration flags (reserved for future use) + .flags = 0, + // Plugin version string (like "1.2.3") + .version = "0.1.0", + // Minimal IPFIXcol version string (like "1.2.3") + .ipx_min = "2.0.0" +}; + +int +ipx_plugin_init(ipx_ctx_t *ctx, const char *xml_config) +{ + std::unique_ptr plugin; + try { + plugin.reset(new Plugin(ctx, xml_config)); + } catch (const std::exception &ex) { + IPX_CTX_ERROR(ctx, "An unexpected exception has occured: %s", ex.what()); + return IPX_ERR_DENIED; + } catch (...) { + IPX_CTX_ERROR(ctx, "An unexpected exception has occured."); + return IPX_ERR_DENIED; + } + ipx_ctx_private_set(ctx, plugin.release()); + return IPX_OK; +} + +void +ipx_plugin_destroy(ipx_ctx_t *ctx, void *priv) +{ + (void) ctx; + Plugin *plugin = reinterpret_cast(priv); + try { + plugin->stop(); + } catch (const std::exception &ex) { + IPX_CTX_ERROR(ctx, "An unexpected exception has occured: %s", ex.what()); + } catch (...) { + IPX_CTX_ERROR(ctx, "An unexpected exception has occured."); + } + delete plugin; +} + +int +ipx_plugin_process(ipx_ctx_t *ctx, void *priv, ipx_msg_t *msg) +{ + Plugin *plugin = reinterpret_cast(priv); + try { + plugin->process(msg); + } catch (const std::exception &ex) { + IPX_CTX_ERROR(ctx, "An unexpected exception has occured: %s", ex.what()); + return IPX_ERR_DENIED; + } catch (...) { + IPX_CTX_ERROR(ctx, "An unexpected exception has occured."); + return IPX_ERR_DENIED; + } + + return IPX_OK; +} diff --git a/src/plugins/output/overview/src/plugin.cpp b/src/plugins/output/overview/src/plugin.cpp new file mode 100644 index 000000000..6242671a1 --- /dev/null +++ b/src/plugins/output/overview/src/plugin.cpp @@ -0,0 +1,119 @@ +/** + * @file + * @author Michal Sedlak + * @brief Main plugin class implementation + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "plugin.h" +#include +#include + +Plugin::Plugin(ipx_ctx_t *ctx, const char *xml_config) + : m_ctx(ctx) + , m_config(parse_config(xml_config)) +{ +} + +void Plugin::process(ipx_msg_t *msg) +{ + if (ipx_msg_get_type(msg) != IPX_MSG_IPFIX) { + return; + } + + ipx_msg_ipfix_t *ipfix_msg = ipx_msg_base2ipfix(msg); + uint32_t drec_cnt = ipx_msg_ipfix_get_drec_cnt(ipfix_msg); + + for (uint32_t i = 0; i < drec_cnt; i++) { + ipx_ipfix_record *drec = ipx_msg_ipfix_get_drec(ipfix_msg, i); + + if (m_config.skip_option_templates && drec->rec.tmplt->type == FDS_TYPE_TEMPLATE_OPTS) { + continue; + } + + fds_drec_iter it; + fds_drec_iter_init(&it, &drec->rec, 0); + + while (fds_drec_iter_next(&it) != FDS_EOC) { + uint64_t key = uint64_t(it.field.info->en) << 16 | uint64_t(it.field.info->id); + auto it = m_field_counts.find(key); + if (it == m_field_counts.end()) { + m_field_counts.emplace(key, 1); + m_fields.emplace_back(key); + } else { + it->second++; + } + } + + m_total_rec_count++; + } +} + +void Plugin::stop() +{ + const fds_iemgr_t *iemgr = ipx_ctx_iemgr_get(m_ctx); + + std::cout << "{\n"; + std::cout << " \"total_number_of_records\": " << m_total_rec_count << ",\n"; + std::cout << " \"elements\": ["; + + bool is_first = true; + + for (const uint64_t field_key : m_fields) { + uint32_t pen = field_key >> 16; + uint16_t id = field_key & 0xFFFF; + uint64_t count = m_field_counts[field_key]; + + const fds_iemgr_elem *elem = fds_iemgr_elem_find_id(iemgr, pen, id); + + if (!is_first) { + std::cout << ","; + } + std::cout << "\n"; + is_first = false; + + std::cout << " {\n"; + std::cout << " \"pen\": " << pen << ",\n"; + std::cout << " \"id\": " << id << ",\n"; + if (elem) { + std::cout << " \"data_type\": \"" << fds_iemgr_type2str(elem->data_type) << "\",\n"; + } else { + std::cout << " \"data_type\": null,\n"; + } + if (elem && elem->scope) { + std::cout << " \"name\": \"" << elem->scope->name << ":" << elem->name << "\",\n"; + } else { + std::cout << " \"name\": null,\n"; // field definition is missing + } + + if (elem) { + std::cout << " \"aliases\": ["; + bool is_first_alias = true; + for (size_t i = 0; i < elem->aliases_cnt; i++) { + const fds_iemgr_alias *alias = elem->aliases[i]; + for (size_t j = 0; j < alias->aliased_names_cnt; j++) { + if (!is_first_alias) { + std::cout << ","; + } + std::cout << " "; + is_first_alias = false; + std::cout << "\"" << alias->aliased_names[j] << "\""; + } + } + std::cout << " "; + std::cout << "],\n"; + } else { + std::cout << " \"aliases\": [ ],\n"; + } + + std::cout << " \"in_number_of_records\": " << count << ",\n"; + std::cout << " \"in_percent_of_records\": " << std::setprecision(2) << (double(count) / m_total_rec_count * 100.0) << "\n"; + std::cout << " }"; + } + std::cout << "\n"; + std::cout << " ]\n"; + std::cout << "}\n"; +} diff --git a/src/plugins/output/overview/src/plugin.h b/src/plugins/output/overview/src/plugin.h new file mode 100644 index 000000000..2a1bb37ee --- /dev/null +++ b/src/plugins/output/overview/src/plugin.h @@ -0,0 +1,52 @@ +/** + * @file + * @author Michal Sedlak + * @brief Main plugin class header file + * @date 2025 + * + * Copyright(c) 2025 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include "config.h" +#include +#include +#include + +class Plugin { +public: + /** + * @brief Instantiate the plugin instance + * + * @param ctx The collector context + * @param xml_config The config in form of a XML string + */ + Plugin(ipx_ctx_t *ctx, const char *xml_config); + + // Disable copy and move + Plugin(Plugin &&) = delete; + Plugin(const Plugin &) = delete; + Plugin &operator=(Plugin &&) = delete; + Plugin &operator=(const Plugin &) = delete; + + /** + * @brief Process a collector message + * + * @param msg The collector message + */ + void process(ipx_msg_t *msg); + + /** + * @brief Stop the plugin and wait till it is stopped (blocking) + */ + void stop(); + +private: + ipx_ctx_t *m_ctx; + Config m_config; + uint64_t m_total_rec_count; + std::map m_field_counts; // (pen, id) -> count + std::vector m_fields; // fields in the order we first saw them +}; From d094dd8e61fcf5bebc772ce4544661637a56fe9f Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Tue, 6 May 2025 16:18:56 +0200 Subject: [PATCH 18/40] Clickhouse - introduce schema helper script --- extra_plugins/output/clickhouse/README.rst | 55 ++- .../output/clickhouse/schema-helper.py | 434 ++++++++++++++++++ 2 files changed, 485 insertions(+), 4 deletions(-) create mode 100644 extra_plugins/output/clickhouse/schema-helper.py diff --git a/extra_plugins/output/clickhouse/README.rst b/extra_plugins/output/clickhouse/README.rst index f8b127af2..5a0a8dea3 100644 --- a/extra_plugins/output/clickhouse/README.rst +++ b/extra_plugins/output/clickhouse/README.rst @@ -32,7 +32,7 @@ To build the plugin, IPFIXcol2 (and its header files) must be installed on your Finally, compile and install the plugin: -.. code-block:: sh +:: $ mkdir build && cd build && cmake .. $ make @@ -47,6 +47,9 @@ and schema of the table is checked after initiating connection to the database and an error is displayed if there is a mismatch. The table is not automatically created. +To aid in initial setup, a script `schema-helper.py` is included. For more information, +see `Schema helper script <#schema-helper>`_. + To run the example configuration below, you can create the ClickHouse table using the following SQL query: @@ -307,9 +310,53 @@ performance at the cost of higher memory usage: .. code-block:: xml - 16 - 128 - 500000 + 16 + 128 + 500000 You can further experiment with the values based on your input characteristics and your machine specifications. + +Schema helper +-------------- + +To aid in initial setup, a script `schema-helper.py` is included. The script +runs ipfixcol2 for a brief period of time to observe structure of the flow data +that is being received, and then generates a ClickHouse schema SQL and a +ipfixcol2 config XML. The generated files can be used as an easier starting +point as opposed to doing everything manually. + +:: + + usage: schema-helper.py [-h] [-a ADDRESS] [-p PORT] [-i INTERVAL] [-t {tcp,udp}] [-s SCHEMA_FILE] [-c CONFIG_FILE] [-o OVERWRITE] + + optional arguments: + -h, --help show this help message and exit + -a ADDRESS, --address ADDRESS + the local IP address, i.e. interface address; empty = all interfaces (default: ) + -p PORT, --port PORT the local port (default: 4739) + -i INTERVAL, --interval INTERVAL + the collection interval in seconds (default: 60) + -t {tcp,udp}, --type {tcp,udp} + the input protocol type (default: udp) + -s SCHEMA_FILE, --schema-file SCHEMA_FILE + the output schema file (default: schema.sql) + -c CONFIG_FILE, --config-file CONFIG_FILE + the output config file (default: config.xml) + -o OVERWRITE, --overwrite OVERWRITE + overwrite the output files without asking if they already exist (default: False) + +**Note:** +In case UDP is used, you might need to increase the interval up to several +minutes for the collector to gather and decode enough data. If you are using +UDP and are getting no results, try running the script with interval set to 300 +or even 600. + + +**Example usage:** + +:: + + $ schema-helper.py -i 60 -p 4739 -t tcp + +Collect data for 60 seconds, listen on port 4739 using TCP. diff --git a/extra_plugins/output/clickhouse/schema-helper.py b/extra_plugins/output/clickhouse/schema-helper.py new file mode 100644 index 000000000..f27c50a5d --- /dev/null +++ b/extra_plugins/output/clickhouse/schema-helper.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python3 + +""" +A helper script to help with ipfixcol2 clickhouse output plugin onboarding. + +Author: Michal Sedlak + +Copyright(c) 2025 CESNET z.s.p.o. +SPDX-License-Identifier: BSD-3-Clause +""" + + +from pathlib import Path +import argparse +import json +import subprocess +import sys +import tempfile +import time + + +RED = "\x1b[0;31m" +GREEN = "\x1b[0;32m" +YELLOW = "\x1b[0;33m" +RESET = "\x1b[0m" + + +IPFIX_TYPE_TO_CLICKHOUSE = { + "unsigned8": "UInt8", + "unsigned16": "UInt16", + "unsigned32": "UInt32", + "unsigned64": "UInt64", + "signed8": "Int8", + "signed16": "Int16", + "signed32": "Int32", + "signed64": "Int64", + "ipv4Address": "IPv4", + "ipv6Address": "IPv6", + "string": "String", + "dateTimeSeconds": "DateTime", + "dateTimeMilliseconds": "DateTime64(3)", + "dateTimeMicroseconds": "DateTime64(6)", + "dateTimeNanoseconds": "DateTime64(9)", + "macAddress": "UInt64", + "float32": "Float32", + "float64": "Float64", + "octetArray": "String", +} + + +class Field: + def __init__(self, name, data_type, note=None, alias_of=[], in_percent_of_records=None): + self.name = name + self.data_type = data_type + self.note = note + self.alias_of = alias_of + self.in_percent_of_records = in_percent_of_records + + def is_(self, name): + return self.name == name or name in self.alias_of + + @property + def comment(self): + s = "" + if self.note: + s += self.note + + if s: + s += "; " + s += f"in {self.in_percent_of_records:.2f}% of records" + + if self.alias_of: + s += f"; alias of {'/'.join(self.alias_of)}" + + return s + + +def have_elem(data, name): + return any(elem["name"] == name for elem in data["elements"]) + + +def is_exclusive_alias(data, alias): + return sum(alias in elem["aliases"] for elem in data["elements"]) == 1 + + +def replace_fields(fields, names, field): + indices = [] + for name in names: + try: + indices.append(next(i for i, field in enumerate(fields) if field.name == name or name in field.alias_of)) + except: + pass + index = min(indices) + + percent = 0.0 + for i in indices: + percent += fields[i].in_percent_of_records + + fields = [field for i, field in enumerate(fields) if i not in indices] + field.in_percent_of_records = percent + fields.insert(index, field) + + return fields + + +def find_field(fields, any_of_names): + for name in any_of_names: + fields = [field for field in fields if field.is_(name)] + if fields: + return fields[0] + return None + + +def generate_config(args, fields): + config = f"""\ + + + + {args.type} input plugin + {args.type} + + {args.port} + {args.address} + + + + + + ClickHouse output + clickhouse + + + + + + CHANGEME.com + 9000 + + + CHANGEME + CHANGEME + CHANGEME + flows
+
+ 16 + 128 + 500000 + true + true + +""" + for field in fields: + config += f"""\ + + {field.name} + {field.name} +""" + if field.comment: + config += f"""\ + +""" + config += """\ + +""" + config += """\ + +
+
+
+
""" + + return config + + +def generate_schema(fields): + flowstart_field = find_field(fields, ["iana:flowStartSeconds", "iana:flowStartMilliseconds", + "iana:flowStartMicroseconds", "iana:flowStartNanoseconds"]) + srcip_field = find_field(fields, ["iana:sourceIPv4Address", "iana:sourceIPv6Address"]) + dstip_field = find_field(fields, ["iana:destinationIPv4Address", "iana:destinationIPv6Address"]) + + schema = "CREATE TABLE flows(\n" + for field, ending in zip(fields, (len(fields) - 1) * [","] + [""]): + schema += f' "{field.name}" {field.data_type}{ending}' + if field.comment: + schema += f" -- {field.comment}" + schema += "\n" + schema += " -- Tip: You may also want to add indexes for better lookup performance\n" + if srcip_field: + schema += f' -- INDEX "{srcip_field.name}_index" "{srcip_field.name}" TYPE bloom_filter GRANULARITY 16,\n' + if dstip_field: + schema += f' -- INDEX "{dstip_field.name}_index" "{dstip_field.name}" TYPE bloom_filter GRANULARITY 16,\n' + if not dstip_field and not srcip_field: + schema += f' -- e.g.: INDEX "srcip_index" "srcip" TYPE bloom_filter GRANULARITY 16,\n' + schema += f' -- INDEX "dstip_index" "dstip" TYPE bloom_filter GRANULARITY 16,\n' + schema += ")\n" + schema += "ENGINE MergeTree\n" + schema += " -- Tip: You may also want to partition and order by flow timestamps\n" + if flowstart_field: + schema += f' -- PARTITION BY toStartOfInterval("{flowstart_field.name}", INTERVAL 1 HOUR)\n' + schema += f' -- ORDER BY "{flowstart_field.name}"\n' + else: + schema += ' -- e.g.: PARTITION BY toStartOfInterval("flowstart", INTERVAL 1 HOUR)\n' + schema += ' -- ORDER BY "flowstart"\n' + return schema.strip() + + +def confirm_yesno(prompt): + while True: + yn = input(f"{YELLOW}{prompt} [y/N] {RESET}").lower().strip() + if yn in ["y", "yes"]: + return True + elif yn in ["n", "no", ""]: + return False + + +def main(): + class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter): + pass + parser = argparse.ArgumentParser(formatter_class=CustomFormatter) + parser.description = """\ +Script to help with ClickHouse output set-up. + +The script runs IPFIXcol2 to collect a sample of data, which is then analyzed +and used for a generation of a ClickHouse schema and a corresponding ClickHouse +output plugin config file. +""" + parser.epilog = """\ +example: + $ schema-helper.py -i 60 -p 4739 -t tcp + Collect data for 60 seconds, listen on port 4739 using TCP. +""" + parser.add_argument("-a", "--address", help="the local IP address, i.e. interface address; empty = all interfaces", type=str, default="") + parser.add_argument("-p", "--port", help="the local port", type=int, default=4739) + parser.add_argument("-i", "--interval", help="the collection interval in seconds", type=int, default=60) + parser.add_argument("-t", "--type", help="the input protocol type", choices=["tcp", "udp"], type=str.lower, default="udp") + parser.add_argument("-s", "--schema-file", help="the output schema file", type=Path, default=Path("schema.sql")) + parser.add_argument("-c", "--config-file", help="the output config file", type=Path, default=Path("config.xml")) + parser.add_argument("-o", "--overwrite", help="overwrite the output files without asking if they already exist", type=bool, default=False) + args = parser.parse_args() + + if subprocess.run("command -v ipfixcol2", shell=True, check=False, capture_output=True).returncode != 0: + print(RED) + print("ERROR: Could not find \"ipfixcol2\" executable.") + print() + print("Is ipfixcol2 installed?") + print(RESET) + sys.exit(1) + + if not "clickhouse" in subprocess.run(["ipfixcol2", "-L"], capture_output=True).stdout.decode(): + print(RED) + print("ERROR: ClickHouse plugin not found.") + print() + print("Ensure your ipfixcol2 is up-to-date and is compiled including \"extra plugins\" (see README).") + print(RESET) + sys.exit(1) + + tmpdir = Path(tempfile.mkdtemp()) + + config = f""" + + + + {args.type} input plugin + {args.type} + + {args.port} + {args.address} + + + + + + Overview output + overview + + true + + + + +""" + (tmpdir/"config.xml").write_text(config) + procargs = ["ipfixcol2", "-c", str(tmpdir / "config.xml")] + print(f"* Starting IPFIXcol2: {' '.join(procargs)}") + proc = subprocess.Popen(procargs, stdout=open(tmpdir / "stdout", "w"), stderr=open(tmpdir / "stderr", "w")) + + print(f"* Waiting {args.interval} seconds", end="\r") + for i in range(1, args.interval + 1): + if proc.poll() is not None: # process exited + break + time.sleep(1) + print(f"* Waiting {args.interval - i} seconds ", end="\r") + print() + + if proc.poll() is None: # process is still running + print("* Stopping IPFIXcol2") + proc.terminate() + proc.wait() + + if proc.returncode != 0: + print(f"{RED}ERROR: IPFIXcol2 process unexpectedly exited with code {proc.returncode}{RESET}.") + print() + print(f"{RED}=== Stderr log ================================================================={RESET}") + print((tmpdir / "stderr").read_text()) + print(f"{RED}================================================================================{RESET}") + print() + sys.exit(1) + + try: + data = json.loads((tmpdir / "stdout").read_text()) + except json.JSONDecodeError as e: + print(f"{RED}ERROR: IPFIXcol2 produced unexpected data. Could not decode JSON: {e}{RESET}") + print() + print(f"{RED}=== Stdout log ================================================================={RESET}") + print((tmpdir / "stdout").read_text()) + print(f"{RED}================================================================================{RESET}") + print() + print(f"{RED}=== Stderr log ================================================================={RESET}") + print((tmpdir / "stderr").read_text()) + print(f"{RED}================================================================================{RESET}") + print() + sys.exit(1) + + if len(data["elements"]) == 0: + print(RED) + print(f"ERROR: No data received.") + print() + print("Ensure the specified port and connection type is correct and that the specified interval is long enough.") + print("If unsure, try increasing the interval (-i).") + if args.type == "udp" and args.interval < 600: + print() + print("In case of UDP, you might want to try to increase the interval up to 300 or even 600.") + print(RESET) + sys.exit(1) + + fields = [] + + fields.append(Field(name="odid", data_type="UInt32", note="maps to the ODID the flow originated from", in_percent_of_records=100.0)) + + for elem in data["elements"]: + if elem["name"] is None: + print(YELLOW, end="") + print(f"WARNING: Element {elem['pen']}:{elem['id']} is missing definition - skipping field") + print(RESET, end="") + continue + + if elem["data_type"] not in IPFIX_TYPE_TO_CLICKHOUSE: + print(YELLOW, end="") + print(f"WARNING: Element {elem['name']}) has unsupported data type {elem['data_type']} - skipping field") + print(RESET, end="") + continue + data_type = IPFIX_TYPE_TO_CLICKHOUSE[elem["data_type"]] + + name = elem["name"] + alias_of = [] + for alias in elem["aliases"]: + # exclude aliases with space in their name as those are inconvenient to use in SQL + if is_exclusive_alias(data, alias) and " " not in alias: + alias_of = [name] + name = alias + break + + fields.append(Field(name=name, data_type=data_type, alias_of=alias_of, in_percent_of_records=elem["in_percent_of_records"])) + + if ((have_elem(data, "iana:sourceIPv4Address") and have_elem(data, "iana:sourceIPv6Address")) + or (have_elem(data, "iana:destinationIPv4Address") and have_elem(data, "iana:destinationIPv6Address"))): + fields = replace_fields( + fields, + ["iana:sourceIPv4Address", "iana:sourceIPv6Address"], + Field(name="srcip", data_type="IPv6", alias_of=["iana:sourceIPv4Address", "iana:sourceIPv6Address"]) + ) + fields = replace_fields( + fields, + ["iana:destinationIPv4Address", "iana:destinationIPv6Address"], + Field(name="dstip", data_type="IPv6", alias_of=["iana:destinationIPv4Address", "iana:destinationIPv6Address"]) + ) + + if (have_elem(data, "iana:flowStartSeconds") + or have_elem(data, "iana:flowStartMilliseconds") + or have_elem(data, "iana:flowStartMicroseconds") + or have_elem(data, "iana:flowStartNanoseconds")): + fields = replace_fields( + fields, + ["iana:flowStartSeconds", "iana:flowStartMilliseconds", "iana:flowStartMicroseconds", "iana:flowStartNanoseconds"], + Field(name="flowstart", data_type="DateTime64(9)", alias_of=["iana:flowStartSeconds", "iana:flowStartMilliseconds", "iana:flowStartMicroseconds", "iana:flowStartNanoseconds"]) + ) + + if (have_elem(data, "iana:flowEndSeconds") + or have_elem(data, "iana:flowEndMilliseconds") + or have_elem(data, "iana:flowEndMicroseconds") + or have_elem(data, "iana:flowEndNanoseconds")): + fields = replace_fields( + fields, + ["iana:flowEndSeconds", "iana:flowEndMilliseconds", "iana:flowEndMicroseconds", "iana:flowEndNanoseconds"], + Field(name="flowend", data_type="DateTime64(9)", alias_of=["iana:flowEndSeconds", "iana:flowEndMilliseconds", "iana:flowEndMicroseconds", "iana:flowEndNanoseconds"]) + ) + + config = generate_config(args, fields) + schema = generate_schema(fields) + + print() + print(f"{GREEN}=== Schema ====================================================================={RESET}") + print(schema) + print(f"{GREEN}================================================================================{RESET}") + write_schema_file = not args.schema_file.exists() or args.overwrite or confirm_yesno(f"{args.schema_file} already exists. Overwrite?") + if write_schema_file: + args.schema_file.write_text(schema) + print(f"Schema has been saved to: {args.schema_file}") + print() + print(f"{GREEN}=== Config ====================================================================={RESET}") + print(config) + print(f"{GREEN}================================================================================{RESET}") + write_config_file = not args.config_file.exists() or args.overwrite or confirm_yesno(f'{args.config_file} already exists. Overwrite?') + if write_config_file: + args.config_file.write_text(config) + print(f"Config has been saved to: {args.config_file}") + print() + + print("You may use the generated schema and config as a starting point to run ipfixcol2 with a ClickHouse database.") + print() + print("- Execute the generated SQL schema to set-up a ClickHouse table that will be used to store the flows.") + print(" e.g. $ clickhouse-client --host 127.0.0.1 --port 9000 --user ipfixcol2 --password ipfixcol2 < schema.sql") + print() + print("- Save the generated XML config to use with ipfixcol2. Make sure to change the example connection parameters.") + print(" e.g. $ ipfixcol2 -c config.xml") + print() + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + # Do not print stack trace on Ctrl+C + sys.exit(1) From bfd5eec92a7e9f53b47a291fcc8db80bbe40f858 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Wed, 11 Jun 2025 16:13:50 +0200 Subject: [PATCH 19/40] Overview - add missing include Missing include would cause FreeBSD build issues (clang) --- src/plugins/output/overview/src/config.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/output/overview/src/config.cpp b/src/plugins/output/overview/src/config.cpp index 7a7dd9734..311694769 100644 --- a/src/plugins/output/overview/src/config.cpp +++ b/src/plugins/output/overview/src/config.cpp @@ -12,8 +12,9 @@ #include -#include #include +#include +#include namespace args { From 6e4ec2a8651c08259b3ac39d7141c4c7679dfbc3 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Wed, 11 Jun 2025 16:30:33 +0200 Subject: [PATCH 20/40] README - fix status badge --- README.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 6e2b3ca33..606785b89 100644 --- a/README.rst +++ b/README.rst @@ -7,8 +7,6 @@ ============= ============= Master branch |BuildMaster| -------------- ------------- -Devel branch |BuildDevel| ============= ============= IPFIXcol2 @@ -201,7 +199,5 @@ be useful also for other users? Please, share your experiences and thoughts. or temporarily change an environment variable "``export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib64/``" -.. |BuildMaster| image:: https://github.com/CESNET/ipfixcol2/workflows/Build%20and%20tests/badge.svg?branch=master +.. |BuildMaster| image:: https://github.com/CESNET/ipfixcol2/actions/workflows/main.yml/badge.svg?branch=master :target: https://github.com/CESNET/ipfixcol2/tree/master -.. |BuildDevel| image:: https://github.com/CESNET/ipfixcol2/workflows/Build%20and%20tests/badge.svg?branch=devel - :target: https://github.com/CESNET/ipfixcol2/tree/devel From 29b8e3c1bcab6a6e6d287ee5bce3152836b7290b Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Wed, 11 Jun 2025 16:43:58 +0200 Subject: [PATCH 21/40] Clickhouse - ignore records using options templates Records using options templates would result in empty rows in the database because no data field would match. Currently we will just ignore them. Sometime in the future, if someone needs it, we can skip empty records to prevent them from being inserted instead and allow for options template records to also be stored. --- extra_plugins/output/clickhouse/src/plugin.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/extra_plugins/output/clickhouse/src/plugin.cpp b/extra_plugins/output/clickhouse/src/plugin.cpp index 61d9acd5e..d521a6a3a 100644 --- a/extra_plugins/output/clickhouse/src/plugin.cpp +++ b/extra_plugins/output/clickhouse/src/plugin.cpp @@ -144,6 +144,13 @@ void Plugin::extract_values(ipx_msg_ipfix_t *msg, RecParser &parser, Block &bloc int Plugin::process_record(ipx_msg_ipfix_t *msg, fds_drec &rec, Block &block) { + if (rec.tmplt->type == FDS_TYPE_TEMPLATE_OPTS) { + // skip the data record if the template used is a options template + // currently we only want data records using "normal" templates + // this is to prevent empty records in the database and might be improved upon in the future + return 0; + } + int ret = 0; RecParser &parser = m_rec_parsers->get_parser(rec.tmplt); parser.parse_record(rec); From 82cdaeb318d6f8a05862a2401274b58091970017 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 6 Jun 2024 15:03:54 +0200 Subject: [PATCH 22/40] core: fix includes in utils.c #include resulted in including a file in which be64toh/htobe64 functions were used. This would clash with the defines present here that are necessary to get the wanted strerror_r function, resulting in the be64toh/htobe64 functions being undefined. This was fixed by not including ipfixcol2.h as a whole, but only the necessary part, as well as adding explicit includes for the system headers that were brought by ipfixcol2.h. --- src/core/utils.c | 5 ++++- src/core/utils.h | 2 -- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core/utils.c b/src/core/utils.c index 1d9049a67..43cf3fe6e 100644 --- a/src/core/utils.c +++ b/src/core/utils.c @@ -46,10 +46,13 @@ #include #include +#include +#include #include #include +#include -#include "utils.h" +#include int diff --git a/src/core/utils.h b/src/core/utils.h index ca4ad63eb..597766973 100644 --- a/src/core/utils.h +++ b/src/core/utils.h @@ -45,8 +45,6 @@ extern "C" { #endif -#include - #ifdef __cplusplus } #endif From f2ff6377aaaf6c3ef73fd6a197be2b82ddcff038 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 19 Jun 2025 14:40:38 +0200 Subject: [PATCH 23/40] fdsdump: add missing include --- src/tools/fdsdump/src/aggregator/stdAllocator.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tools/fdsdump/src/aggregator/stdAllocator.hpp b/src/tools/fdsdump/src/aggregator/stdAllocator.hpp index 3414f1050..7f172a8d7 100644 --- a/src/tools/fdsdump/src/aggregator/stdAllocator.hpp +++ b/src/tools/fdsdump/src/aggregator/stdAllocator.hpp @@ -9,6 +9,7 @@ #pragma once +#include #include #include From 9e814d8059032b9b22382b52fd63578c008618a6 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Thu, 19 Jun 2025 15:27:30 +0200 Subject: [PATCH 24/40] core: fix includes in utils.c Fix of previous fix... --- src/core/utils.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/core/utils.c b/src/core/utils.c index 43cf3fe6e..e6240c68c 100644 --- a/src/core/utils.c +++ b/src/core/utils.c @@ -52,7 +52,14 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif #include +#include +#ifdef __cplusplus +} +#endif int From 6061caf4831947b20e5e72cc945c9c7c2717234f Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Wed, 2 Jul 2025 10:33:58 +0200 Subject: [PATCH 25/40] Update copyright years in LICENSE / ipfixcol2 -V --- LICENSE | 2 +- src/core/main.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 6ece728a6..33e14b899 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (C) 2015-2017 CESNET, z.s.p.o. +Copyright (C) 2015-2025 CESNET, z.s.p.o. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions diff --git a/src/core/main.cpp b/src/core/main.cpp index 1c133b9e5..c79daec4f 100644 --- a/src/core/main.cpp +++ b/src/core/main.cpp @@ -73,7 +73,7 @@ print_version() << "Build type: " << IPX_BUILD_TYPE << "\n" << "Architecture: " << IPX_BUILD_ARCH << " (" << IPX_BUILD_BYTE_ORDER << ")" << "\n" << "Compiler: " << IPX_BUILD_COMPILER << "\n" - << "Copyright (C) 2018 CESNET z.s.p.o.\n"; + << "Copyright (C) 2025 CESNET z.s.p.o.\n"; } /** From 5cc285760f56221932b2e07f87c9692ab685cdc3 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Wed, 2 Jul 2025 10:32:47 +0200 Subject: [PATCH 26/40] Forwarder: fix templates being repeatedly sent when using TCP When using UDP, templates are periodically resent unlike TCP, where this periodic resending should not happen. Because of a missing check this periodic resending was happening even over TCP. --- src/plugins/output/forwarder/src/Connection.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/output/forwarder/src/Connection.cpp b/src/plugins/output/forwarder/src/Connection.cpp index 9caaf11aa..9ebd0e658 100644 --- a/src/plugins/output/forwarder/src/Connection.cpp +++ b/src/plugins/output/forwarder/src/Connection.cpp @@ -236,8 +236,8 @@ Connection::get_or_create_sender(ipx_msg_ipfix_t *msg) send_message(msg); }, m_con_params.protocol == Protocol::TCP, - m_tmplts_resend_pkts, - m_tmplts_resend_secs))); + m_con_params.protocol != Protocol::TCP ? m_tmplts_resend_pkts : 0, + m_con_params.protocol != Protocol::TCP ? m_tmplts_resend_secs : 0))); } Sender &sender = *m_senders[odid].get(); From 30bb2c86d9616bd03cb0f619b1e5b7f1aa3cf1c8 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Wed, 2 Jul 2025 13:27:16 +0200 Subject: [PATCH 27/40] Forwarder: handle periodic message Adds support for newly introduced "periodic message". Without this change, unfinished data transfers might never finish if no input data is coming. --- src/plugins/output/forwarder/src/Forwarder.cpp | 8 ++++++++ src/plugins/output/forwarder/src/Forwarder.h | 10 +++++++++- src/plugins/output/forwarder/src/Host.cpp | 11 +++++++++++ src/plugins/output/forwarder/src/Host.h | 8 +++++++- src/plugins/output/forwarder/src/main.cpp | 6 +++++- 5 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/plugins/output/forwarder/src/Forwarder.cpp b/src/plugins/output/forwarder/src/Forwarder.cpp index da0e89f11..318caf114 100644 --- a/src/plugins/output/forwarder/src/Forwarder.cpp +++ b/src/plugins/output/forwarder/src/Forwarder.cpp @@ -105,6 +105,14 @@ void Forwarder::handle_ipfix_message(ipx_msg_ipfix_t *msg) } } +void +Forwarder::handle_periodic_message(ipx_msg_periodic_t *msg) +{ + for (auto &host : m_hosts) { + host->advance_transfers(); + } +} + void Forwarder::forward_to_all(ipx_msg_ipfix_t *msg) { diff --git a/src/plugins/output/forwarder/src/Forwarder.h b/src/plugins/output/forwarder/src/Forwarder.h index 2fc11dddf..2aba16776 100644 --- a/src/plugins/output/forwarder/src/Forwarder.h +++ b/src/plugins/output/forwarder/src/Forwarder.h @@ -80,6 +80,13 @@ class Forwarder { void handle_ipfix_message(ipx_msg_ipfix_t *msg); + /** + * \brief Handle a periodic message + * \param msg The periodic message + */ + void + handle_periodic_message(ipx_msg_periodic_t *msg); + /** * \brief The destructor - finalize the forwarder */ @@ -101,4 +108,5 @@ class Forwarder { void forward_round_robin(ipx_msg_ipfix_t *msg); -}; \ No newline at end of file + +}; diff --git a/src/plugins/output/forwarder/src/Host.cpp b/src/plugins/output/forwarder/src/Host.cpp index 26851e7ec..a44a5fa46 100644 --- a/src/plugins/output/forwarder/src/Host.cpp +++ b/src/plugins/output/forwarder/src/Host.cpp @@ -142,6 +142,17 @@ Host::forward_message(ipx_msg_ipfix_t *msg) return true; } +void +Host::advance_transfers() +{ + for (auto &p : m_session_to_connection) { + Connection &connection = *p.second.get(); + if (connection.check_connected()) { + connection.advance_transfers(); + } + } +} + Host::~Host() { for (auto &p : m_session_to_connection) { diff --git a/src/plugins/output/forwarder/src/Host.h b/src/plugins/output/forwarder/src/Host.h index 4189f5669..8868b6ece 100644 --- a/src/plugins/output/forwarder/src/Host.h +++ b/src/plugins/output/forwarder/src/Host.h @@ -100,6 +100,12 @@ class Host { bool forward_message(ipx_msg_ipfix_t *msg); + /** + * \brief Advance the unfinished transfers + */ + void + advance_transfers(); + private: const std::string &m_ident; @@ -116,4 +122,4 @@ class Host { Connector &m_connector; std::unordered_map> m_session_to_connection; -}; \ No newline at end of file +}; diff --git a/src/plugins/output/forwarder/src/main.cpp b/src/plugins/output/forwarder/src/main.cpp index a05d61fdf..59dc35e8c 100644 --- a/src/plugins/output/forwarder/src/main.cpp +++ b/src/plugins/output/forwarder/src/main.cpp @@ -88,7 +88,7 @@ ipx_plugin_init(ipx_ctx_t *ctx, const char *xml_config) return IPX_ERR_DENIED; } - ipx_msg_mask_t mask = IPX_MSG_IPFIX | IPX_MSG_SESSION; + ipx_msg_mask_t mask = IPX_MSG_IPFIX | IPX_MSG_SESSION | IPX_MSG_PERIODIC; ipx_ctx_subscribe(ctx, &mask, NULL); ipx_ctx_private_set(ctx, forwarder.release()); @@ -121,6 +121,10 @@ ipx_plugin_process(ipx_ctx_t *ctx, void *priv, ipx_msg_t *msg) forwarder.handle_session_message(ipx_msg_base2session(msg)); break; + case IPX_MSG_PERIODIC: + forwarder.handle_periodic_message(ipx_msg_base2periodic(msg)); + break; + default: assert(0); } From 68ec5a82fd6d61cf857640419334f0648fab1848 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Wed, 2 Jul 2025 13:31:34 +0200 Subject: [PATCH 28/40] Forwarder: clear buffers on connection lost Unfinished transfers would remain in buffers when we lost connection. In case of TCP, this is a problem. A message might be half-way send when the connection drops, resulting in the other half being sent after the TCP connection reestabilishes. The collector would interpret this as an entirely new message, resulting in bogus data being decoded. --- .../output/forwarder/src/Connection.cpp | 53 ++++++++++++------- src/plugins/output/forwarder/src/Connection.h | 5 +- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/src/plugins/output/forwarder/src/Connection.cpp b/src/plugins/output/forwarder/src/Connection.cpp index 9ebd0e658..367189828 100644 --- a/src/plugins/output/forwarder/src/Connection.cpp +++ b/src/plugins/output/forwarder/src/Connection.cpp @@ -87,8 +87,7 @@ Connection::forward_message(ipx_msg_ipfix_t *msg) sender.process_message(msg); } catch (const ConnectionError &err) { - // In case connection was lost, we have to resend templates when it reconnects - sender.clear_templates(); + on_connection_lost(); throw err; } } @@ -107,31 +106,36 @@ Connection::advance_transfers() IPX_CTX_DEBUG(m_log_ctx, "Waiting transfers on connection %s: %zu", m_ident.c_str(), m_transfers.size()); - for (auto it = m_transfers.begin(); it != m_transfers.end(); ) { + try { + for (auto it = m_transfers.begin(); it != m_transfers.end(); ) { - Transfer &transfer = *it; + Transfer &transfer = *it; - assert(transfer.data.size() <= UINT16_MAX); // The transfer consists of one IPFIX message which cannot be larger + assert(transfer.data.size() <= UINT16_MAX); // The transfer consists of one IPFIX message which cannot be larger - ssize_t ret = send(m_sockfd.get(), &transfer.data[transfer.offset], - transfer.data.size() - transfer.offset, MSG_DONTWAIT | MSG_NOSIGNAL); + ssize_t ret = send(m_sockfd.get(), &transfer.data[transfer.offset], + transfer.data.size() - transfer.offset, MSG_DONTWAIT | MSG_NOSIGNAL); - check_socket_error(ret); + check_socket_error(ret); - size_t sent = std::max(0, ret); - IPX_CTX_DEBUG(m_log_ctx, "Sent %zu/%zu B to %s", sent, transfer.data.size(), m_ident.c_str()); + size_t sent = std::max(0, ret); + IPX_CTX_DEBUG(m_log_ctx, "Sent %zu/%zu B to %s", sent, transfer.data.size(), m_ident.c_str()); - // Is the transfer done? - if (transfer.offset + sent == transfer.data.size()) { - it = m_transfers.erase(it); - // Remove the transfer and continue with the next one + // Is the transfer done? + if (transfer.offset + sent == transfer.data.size()) { + it = m_transfers.erase(it); + // Remove the transfer and continue with the next one - } else { - transfer.offset += sent; + } else { + transfer.offset += sent; - // Finish, cannot advance next transfer before the one before it is fully sent - break; + // Finish, cannot advance next transfer before the one before it is fully sent + break; + } } + } catch (ConnectionError& err) { + on_connection_lost(); + throw err; } } @@ -261,3 +265,16 @@ Connection::check_socket_error(ssize_t sock_ret) throw ConnectionError(errbuf); } } + +void +Connection::on_connection_lost() +{ + for (auto& p : m_senders) { + // In case connection was lost, we have to resend templates when it reconnects + Sender& sender = *p.second.get(); + sender.clear_templates(); + } + + // Do not continue any of the transfers that haven't been finished so we don't end up in the middle of a message + m_transfers.clear(); +} diff --git a/src/plugins/output/forwarder/src/Connection.h b/src/plugins/output/forwarder/src/Connection.h index 02c76f1de..d137b5c46 100644 --- a/src/plugins/output/forwarder/src/Connection.h +++ b/src/plugins/output/forwarder/src/Connection.h @@ -181,4 +181,7 @@ class Connection { void check_socket_error(ssize_t sock_ret); -}; \ No newline at end of file + + void + on_connection_lost(); +}; From 1f53cead4ffb0bb89d6994238947c38d50caedd7 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Mon, 11 Aug 2025 11:42:51 +0200 Subject: [PATCH 29/40] fdsdump: allow loading filter from stdin instead of command line --- src/tools/fdsdump/src/options.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/tools/fdsdump/src/options.cpp b/src/tools/fdsdump/src/options.cpp index 11231ad2e..4b3544bfa 100644 --- a/src/tools/fdsdump/src/options.cpp +++ b/src/tools/fdsdump/src/options.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -133,7 +134,15 @@ void Options::parse(int argc, char *argv[]) } if (args.has('F')) { - m_input_filter = args.get('F'); + if (args.get('F') == "-") { + std::string line; + while (std::getline(std::cin, line)) { + m_input_filter.append(line); + m_input_filter.append("\n"); + } + } else { + m_input_filter = args.get('F'); + } } if (args.has('A')) { From 021bd292ad6200a4cc79ba2856c7fc6f4dc756a8 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Tue, 27 Jan 2026 16:33:38 +0100 Subject: [PATCH 30/40] core: fix includes in utils.c Missing include was causing a symbol to not be exported. --- src/core/utils.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/utils.c b/src/core/utils.c index e6240c68c..81c5c1b68 100644 --- a/src/core/utils.c +++ b/src/core/utils.c @@ -57,6 +57,7 @@ extern "C" { #endif #include #include +#include #ifdef __cplusplus } #endif From 4cef161f99b4ed441651aeaf0df28b04086848cd Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Wed, 4 Mar 2026 23:18:16 +0100 Subject: [PATCH 31/40] ci: remove debian buster (EOL) --- .github/workflows/main.yml | 1 - .github/workflows/packages.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index aae1e191e..cedb546a7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,7 +18,6 @@ jobs: image: - 'ubuntu:20.04' - 'ubuntu:22.04' - - 'debian:buster' - 'debian:bullseye' - 'debian:bookworm' - 'rockylinux:8' diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 8e616e1ed..1829b6054 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - image: ['ubuntu:20.04', 'ubuntu:22.04', 'debian:buster', 'debian:bullseye', 'debian:bookworm'] + image: ['ubuntu:20.04', 'ubuntu:22.04', 'debian:bullseye', 'debian:bookworm'] name: Build DEBs on ${{ matrix.image }} container: ${{ matrix.image }} From 616018c395f917ec4cc08298cfd59261502a4cf3 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Wed, 4 Mar 2026 23:15:25 +0100 Subject: [PATCH 32/40] ci: update freebsd 14.2 (EOL) -> 14.3 (new stable) --- .github/workflows/freebsd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml index e39ae1a2b..ea61c6e4b 100644 --- a/.github/workflows/freebsd.yml +++ b/.github/workflows/freebsd.yml @@ -15,10 +15,10 @@ jobs: name: Build and run tests on FreeBSD steps: - uses: actions/checkout@v4 - - name: Build on FreeBSD 14.2 + - name: Build on FreeBSD 14.3 uses: vmactions/freebsd-vm@v1 with: - release: "14.2" + release: "14.3" usesh: true run: | set -euox pipefail From 5b864513bce73ef8d6c9bfd658c043502cf64d40 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Wed, 4 Mar 2026 23:31:21 +0100 Subject: [PATCH 33/40] ci: add ubuntu 24.04 --- .github/workflows/main.yml | 1 + .github/workflows/packages.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cedb546a7..5d0747792 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,6 +18,7 @@ jobs: image: - 'ubuntu:20.04' - 'ubuntu:22.04' + - 'ubuntu:24.04' - 'debian:bullseye' - 'debian:bookworm' - 'rockylinux:8' diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 1829b6054..4cebc9550 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - image: ['ubuntu:20.04', 'ubuntu:22.04', 'debian:bullseye', 'debian:bookworm'] + image: ['ubuntu:20.04', 'ubuntu:22.04', 'ubuntu:24.04', 'debian:bullseye', 'debian:bookworm'] name: Build DEBs on ${{ matrix.image }} container: ${{ matrix.image }} From 3bae3abd881d6ee765942cdecd2a3710f5b78e22 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Wed, 4 Mar 2026 23:33:38 +0100 Subject: [PATCH 34/40] ci: add oracle linux 10 --- .github/workflows/main.yml | 10 ++++++++++ .github/workflows/packages.yml | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5d0747792..1a599aaac 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,6 +24,7 @@ jobs: - 'rockylinux:8' - 'oraclelinux:8' - 'oraclelinux:9' + - 'oraclelinux:10' name: Build on ${{ matrix.image }} container: ${{ matrix.image }} @@ -54,6 +55,11 @@ jobs: run: | dnf -y install 'dnf-command(config-manager)' dnf config-manager --set-enabled ol9_appstream ol9_codeready_builder + - name: Enable additional repositories (Oracle Linux 10) + if: contains(matrix.image, 'oraclelinux:10') + run: | + dnf -y install 'dnf-command(config-manager)' + dnf config-manager --set-enabled ol10_appstream ol10_codeready_builder - name: Enable EPEL (Rocky Linux) if: contains(matrix.image, 'rockylinux') run: | @@ -66,6 +72,10 @@ jobs: if: contains(matrix.image, 'oraclelinux:9') run: | dnf -y install oracle-epel-release-el9 + - name: Enable EPEL (Oracle Linux 10) + if: contains(matrix.image, 'oraclelinux:10') + run: | + dnf -y install oracle-epel-release-el10 - name: Install dependencies for libfds and IPFIXcol2 (Rocky Linux, Oracle Linux) if: contains(matrix.image, 'rockylinux') || contains(matrix.image, 'oraclelinux') run: | diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 4cebc9550..efb3b1588 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -128,6 +128,11 @@ jobs: run: | dnf -y install 'dnf-command(config-manager)' dnf config-manager --set-enabled ol9_appstream ol9_codeready_builder + - name: Enable additional repositories (Oracle Linux 10) + if: contains(matrix.image, 'oraclelinux:10') + run: | + dnf -y install 'dnf-command(config-manager)' + dnf config-manager --set-enabled ol10_appstream ol10_codeready_builder - name: Enable EPEL (Rocky Linux) if: contains(matrix.image, 'Rocky Linux') run: | @@ -140,6 +145,10 @@ jobs: if: contains(matrix.image, 'oraclelinux:9') run: | dnf -y install oracle-epel-release-el9 + - name: Enable EPEL (Oracle Linux 10) + if: contains(matrix.image, 'oraclelinux:10') + run: | + dnf -y install oracle-epel-release-el10 - name: Install dependencies for libfds and IPFIXcol2 (Rocky Linux, Oracle Linux) if: contains(matrix.image, 'rockylinux') || contains(matrix.image, 'oraclelinux') run: | From c7c53b7bed97cb246fb02abca4716dd0aed3ef71 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Tue, 19 May 2026 12:26:28 +0200 Subject: [PATCH 35/40] cmake: add -rdynamic flag to fix "undefined symbol" issue starting from ubuntu 26.04 This would happen when loading plugin .so. This was likely the default before, or it is needed now because of more aggresive linking optimizations. Either way, according to the docs, it should be used when dealing with exactly this use-case. --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 366cc8098..a14ffbc92 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,10 +65,10 @@ option(PACKAGE_BUILDER_DEB "Enable DEB package builder (make deb)" OFF) ## ----------------------------------------------------------------------------- # Hard coded definitions -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden -std=gnu11") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden -std=gnu11 -rdynamic") set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG") set(CMAKE_C_FLAGS_DEBUG "-g -O0 -Wall -Wextra -pedantic") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden -std=gnu++11") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden -std=gnu++11 -rdynamic") set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG") set(CMAKE_CXX_FLAGS_DEBUG "-g -O0 -Wall -Wextra -pedantic") From 636fed7dbf7e9f21de54515a0fefb320c38d1d8e Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Wed, 1 Jul 2026 12:00:59 +0200 Subject: [PATCH 36/40] fdsdump: dont assume we are on 64bit architecture when formatting time --- src/tools/fdsdump/src/aggregator/print.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/tools/fdsdump/src/aggregator/print.cpp b/src/tools/fdsdump/src/aggregator/print.cpp index b861f0d9f..19fba9df1 100644 --- a/src/tools/fdsdump/src/aggregator/print.cpp +++ b/src/tools/fdsdump/src/aggregator/print.cpp @@ -15,6 +15,7 @@ #include #include +#include #include namespace fdsdump { @@ -61,7 +62,6 @@ std::string datetime_to_str(uint64_t ts_millisecs) { char buffer[128]; - static_assert(sizeof(uint64_t) == sizeof(time_t), "Assumed that time_t is uint64_t, but it's not"); uint64_t secs = ts_millisecs / 1000; uint64_t msecs_part = ts_millisecs % 1000; @@ -71,13 +71,12 @@ datetime_to_str(uint64_t ts_millisecs) msecs_part *= 10; } + time_t t_secs = std::min(secs, std::numeric_limits::max()); tm tm; - localtime_r(reinterpret_cast(&secs), &tm); - //TODO: The format should probably be configrable - //TODO: Have a look at the hard coded buffer size + localtime_r(&t_secs, &tm); std::size_t n = strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &tm); assert(n > 0); - snprintf(&buffer[n], sizeof(buffer) - n, ".%03lu", msecs_part); + snprintf(&buffer[n], sizeof(buffer) - n, ".%03" PRIu64, msecs_part); return buffer; } From 1a199033eff847941353c5c039cb806a9ad8aa6f Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Wed, 14 Oct 2026 14:13:36 +0200 Subject: [PATCH 37/40] TCP input: add limit for maximum buffer size in Lz4 decoder --- src/plugins/input/tcp/src/Lz4Decoder.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/plugins/input/tcp/src/Lz4Decoder.cpp b/src/plugins/input/tcp/src/Lz4Decoder.cpp index 6b8a64e79..6598ca5b9 100644 --- a/src/plugins/input/tcp/src/Lz4Decoder.cpp +++ b/src/plugins/input/tcp/src/Lz4Decoder.cpp @@ -25,6 +25,9 @@ #include "DecodeBuffer.hpp" // DecodeBuffer #include "read_until_n.hpp" // read_until_n +// This is the maximum size of the buffer used for receiving compressed data and then decompressing it. +static constexpr size_t LZ4_MAX_BUFFER_SIZE = 1 * 1024 * 1024; + namespace tcp_in { struct __attribute__((__packed__)) ipfix_compress_header { @@ -124,6 +127,9 @@ bool Lz4Decoder::read_start_header() { // Get the ring buffer size from the header. auto hdr = reinterpret_cast(m_compressed.data()); auto new_buffer_size = ntohl(hdr->buffer_size); + if (new_buffer_size == 0 || new_buffer_size > LZ4_MAX_BUFFER_SIZE) { + throw std::runtime_error("LZ4 Decoder: invalid buffer size"); + } // The start header is not part of the message that should be decompressed. m_compressed.clear(); From fd5cb168efb4b864d33f655174711b2779cef502 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Wed, 14 Oct 2026 14:13:03 +0200 Subject: [PATCH 38/40] TCP input: fix possible buffer overflow in Lz4 decoder --- src/plugins/input/tcp/src/Lz4Decoder.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/plugins/input/tcp/src/Lz4Decoder.cpp b/src/plugins/input/tcp/src/Lz4Decoder.cpp index 6598ca5b9..7024396d2 100644 --- a/src/plugins/input/tcp/src/Lz4Decoder.cpp +++ b/src/plugins/input/tcp/src/Lz4Decoder.cpp @@ -148,6 +148,10 @@ void Lz4Decoder::decompress() { m_decompressed_pos = 0; } + if (m_decompressed_size > m_decompressed.size() - m_decompressed_pos) { + throw std::runtime_error("LZ4 Decoder: invalid decompressed size"); + } + int res = LZ4_decompress_safe_continue( m_decoder.get(), reinterpret_cast(m_compressed.data()), From ad206f6217257b06b139f1b2bce50d52f5bc6acb Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Fri, 16 Oct 2026 15:29:52 +0200 Subject: [PATCH 39/40] Forwarder output: fix possible buffer overflow --- src/plugins/output/forwarder/src/Message.cpp | 15 +++++++++++-- src/plugins/output/forwarder/src/Message.h | 8 +++---- src/plugins/output/forwarder/src/Sender.cpp | 23 ++++++++++++++------ 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/plugins/output/forwarder/src/Message.cpp b/src/plugins/output/forwarder/src/Message.cpp index 08229dd7b..ee9967c18 100644 --- a/src/plugins/output/forwarder/src/Message.cpp +++ b/src/plugins/output/forwarder/src/Message.cpp @@ -41,6 +41,7 @@ #include "Message.h" #include +#include static uint16_t get_set_id(fds_template_type template_type) @@ -98,6 +99,7 @@ void Message::finalize() { finalize_set(); + assert(m_length <= UINT16_MAX); m_msg_hdr->length = htons(m_length); } @@ -119,6 +121,10 @@ Message::add_part(uint8_t *data, uint16_t length) { assert(data < m_buffer || data >= m_buffer + BUFFER_SIZE); + if (UINT16_MAX - m_length < length) { + throw std::logic_error("message exceeds maximum size"); + } + iovec part; part.iov_base = data; part.iov_len = length; @@ -132,7 +138,12 @@ Message::add_part(uint8_t *data, uint16_t length) uint8_t * Message::write(const uint8_t *data, uint16_t length) { - assert(BUFFER_SIZE - m_buffer_pos >= length); + if (BUFFER_SIZE - m_buffer_pos < length) { + throw std::logic_error("message buffer full"); + } + if (UINT16_MAX - m_length < length) { + throw std::logic_error("message exceeds maximum size"); + } uint8_t *p = &m_buffer[m_buffer_pos]; m_buffer_pos += length; @@ -182,4 +193,4 @@ Message::finalize_set() m_current_set_hdr->flowset_id = htons(m_current_set_hdr->flowset_id); m_current_set_hdr->length = htons(m_current_set_hdr->length); m_current_set_hdr = nullptr; -} \ No newline at end of file +} diff --git a/src/plugins/output/forwarder/src/Message.h b/src/plugins/output/forwarder/src/Message.h index 890808c69..3373fa6bd 100644 --- a/src/plugins/output/forwarder/src/Message.h +++ b/src/plugins/output/forwarder/src/Message.h @@ -110,7 +110,7 @@ class Message { * \brief Get the total length of the message * \return The length */ - uint16_t length() const { return m_length; } + size_t length() const { return m_length; } /** * \brief Check if the message is empty or only contains headers but no content @@ -127,13 +127,13 @@ class Message { private: static constexpr uint16_t BUFFER_SIZE = UINT16_MAX; - std::vector m_parts; + std::vector m_parts; - uint16_t m_length = 0; + size_t m_length = 0; uint8_t m_buffer[BUFFER_SIZE]; //NOTE: Maybe this should be dynamically expanding instead? - uint16_t m_buffer_pos = 0; + size_t m_buffer_pos = 0; fds_ipfix_msg_hdr *m_msg_hdr = nullptr; diff --git a/src/plugins/output/forwarder/src/Sender.cpp b/src/plugins/output/forwarder/src/Sender.cpp index 635da33c4..f172e8f4b 100644 --- a/src/plugins/output/forwarder/src/Sender.cpp +++ b/src/plugins/output/forwarder/src/Sender.cpp @@ -42,6 +42,8 @@ #include "Sender.h" #include +static constexpr const size_t TMPLT_WDRL_ALL_SIZE = 16; + class DrecsForwardIterator { public: DrecsForwardIterator(ipx_msg_ipfix_t *msg) : m_msg(msg) {} @@ -206,20 +208,27 @@ Sender::lose_message(ipx_msg_ipfix_t *msg) void Sender::process_templates(const fds_tsnapshot_t *tsnap, uint32_t next_seq_num) { + auto emit_and_start_new_message = [&]() { + m_message.finalize(); + + emit_message(); + + fds_ipfix_msg_hdr msg_hdr = *m_message.header(); + msg_hdr.seq_num = htonl(next_seq_num); + m_message.start(&msg_hdr); + }; + if (m_do_withdrawals) { + if (m_message.length() + TMPLT_WDRL_ALL_SIZE > TMPLTMSG_MAX_LENGTH && !m_message.empty()) { + emit_and_start_new_message(); + } m_message.add_template_withdrawal_all(); } tsnapshot_for_each(tsnap, [&](const fds_template *tmplt) { // Should we start another message? if (m_message.length() + sizeof(fds_ipfix_set_hdr) + tmplt->raw.length > TMPLTMSG_MAX_LENGTH && !m_message.empty()) { - m_message.finalize(); - - emit_message(); - - fds_ipfix_msg_hdr msg_hdr = *m_message.header(); - msg_hdr.seq_num = htonl(next_seq_num); - m_message.start(&msg_hdr); + emit_and_start_new_message(); } m_message.add_template(tmplt); From c731deb5486f8b8aaa1ff476d833bf8418130118 Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Fri, 16 Oct 2026 18:09:21 +0200 Subject: [PATCH 40/40] Bump version to 2.9.0 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a14ffbc92..053c6e0d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,7 @@ endif() # Versions and other informations set(IPFIXCOL_VERSION_MAJOR 2) -set(IPFIXCOL_VERSION_MINOR 8) +set(IPFIXCOL_VERSION_MINOR 9) set(IPFIXCOL_VERSION_PATCH 0) set(IPFIXCOL_VERSION ${IPFIXCOL_VERSION_MAJOR}.${IPFIXCOL_VERSION_MINOR}.${IPFIXCOL_VERSION_PATCH})