diff --git a/include/fluent-bit/flb_log.h b/include/fluent-bit/flb_log.h index 4d4d1a78cec..11fb4831326 100644 --- a/include/fluent-bit/flb_log.h +++ b/include/fluent-bit/flb_log.h @@ -124,6 +124,7 @@ struct flb_log { struct flb_log_cache_entry { flb_sds_t buf; uint64_t timestamp; + int interval; struct mk_list _head; }; @@ -193,6 +194,9 @@ struct flb_log_cache_entry *flb_log_cache_exists(struct flb_log_cache *cache, ch struct flb_log_cache_entry *flb_log_cache_get_target(struct flb_log_cache *cache, uint64_t ts); int flb_log_cache_check_suppress(struct flb_log_cache *cache, char *msg_buf, size_t msg_size); +int flb_log_cache_check_suppress_interval(struct flb_log_cache *cache, + char *msg_buf, size_t msg_size, + int interval_seconds); static inline int flb_log_suppress_check(int log_suppress_interval, const char *fmt, ...) @@ -220,7 +224,8 @@ static inline int flb_log_suppress_check(int log_suppress_interval, const char * return FLB_FALSE; } - ret = flb_log_cache_check_suppress(w->log_cache, buf, size); + ret = flb_log_cache_check_suppress_interval(w->log_cache, buf, size, + log_suppress_interval); return ret; } diff --git a/src/flb_log.c b/src/flb_log.c index 8fb090737c9..35aeaee67bd 100644 --- a/src/flb_log.c +++ b/src/flb_log.c @@ -676,7 +676,7 @@ struct flb_log_cache_entry *flb_log_cache_get_target(struct flb_log_cache *cache } /* expired entry */ - if (entry->timestamp + cache->timeout < ts) { + if (entry->timestamp + entry->interval <= ts) { return entry; } @@ -695,7 +695,9 @@ struct flb_log_cache_entry *flb_log_cache_get_target(struct flb_log_cache *cache * * if no similar message exists, then the incoming message is added to the cache. */ -int flb_log_cache_check_suppress(struct flb_log_cache *cache, char *msg_buf, size_t msg_size) +int flb_log_cache_check_suppress_interval(struct flb_log_cache *cache, + char *msg_buf, size_t msg_size, + int interval_seconds) { uint64_t now = 0; flb_sds_t buf; @@ -726,20 +728,30 @@ int flb_log_cache_check_suppress(struct flb_log_cache *cache, char *msg_buf, siz entry->buf = buf; entry->timestamp = now; + entry->interval = interval_seconds; return FLB_FALSE; } else { - if (entry->timestamp + cache->timeout > now) { + if (entry->timestamp + interval_seconds > now) { + entry->interval = interval_seconds; return FLB_TRUE; } else { entry->timestamp = now; + entry->interval = interval_seconds; return FLB_FALSE; } } return FLB_TRUE; } +int flb_log_cache_check_suppress(struct flb_log_cache *cache, + char *msg_buf, size_t msg_size) +{ + return flb_log_cache_check_suppress_interval(cache, msg_buf, msg_size, + cache->timeout); +} + int flb_log_worker_destroy(struct flb_worker *worker) { (void) worker; diff --git a/tests/internal/log.c b/tests/internal/log.c index 10b09fb8b49..c3ab534e91a 100644 --- a/tests/internal/log.c +++ b/tests/internal/log.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include "flb_tests_internal.h" @@ -13,6 +14,9 @@ #define TEST_RECORD_02 "other type of message" #define TEST_RECORD_02_SIZE sizeof(TEST_RECORD_02) - 1 +#define TEST_RECORD_03 "third type of message" +#define TEST_RECORD_03_SIZE sizeof(TEST_RECORD_03) - 1 + static int check_clock(uint64_t timeout, struct flb_time *tm_start) { struct flb_time tm_now; @@ -164,8 +168,118 @@ static void cache_one_slot() flb_log_cache_destroy(cache); } +static void check_suppress_interval(int cache_timeout, int suppress_interval, + int message_age, int expected) +{ + int ret; + struct flb_worker worker = {0}; + struct flb_worker *previous_worker; + struct flb_log_cache_entry *entry; + + worker.log_cache = flb_log_cache_create(cache_timeout, 1); + TEST_CHECK(worker.log_cache != NULL); + if (!worker.log_cache) { + return; + } + + previous_worker = flb_worker_get(); + FLB_TLS_SET(flb_worker_ctx, &worker); + + ret = flb_log_suppress_check(suppress_interval, TEST_RECORD_01); + TEST_CHECK(ret == FLB_FALSE); + + entry = flb_log_cache_exists(worker.log_cache, + TEST_RECORD_01, TEST_RECORD_01_SIZE); + TEST_CHECK(entry != NULL); + if (entry) { + entry->timestamp = time(NULL) - message_age; + } + + ret = flb_log_suppress_check(suppress_interval, TEST_RECORD_01); + if (!TEST_CHECK(ret == expected)) { + TEST_MSG("cache timeout=%d, suppress interval=%d, message age=%d, " + "expected=%d, actual=%d", + cache_timeout, suppress_interval, message_age, expected, ret); + } + + FLB_TLS_SET(flb_worker_ctx, previous_worker); + flb_log_cache_destroy(worker.log_cache); +} + +static void suppress_interval_longer_than_cache_timeout() +{ + /* A two-second-old message is still within the configured interval. */ + check_suppress_interval(1, 3, 2, FLB_TRUE); +} + +static void suppress_interval_shorter_than_cache_timeout() +{ + /* A two-second-old message is outside the configured interval. */ + check_suppress_interval(3, 1, 2, FLB_FALSE); +} + +static void suppress_interval_preserved_during_cache_replacement() +{ + int ret; + struct flb_worker worker = {0}; + struct flb_worker *previous_worker; + struct flb_log_cache_entry *entry; + + worker.log_cache = flb_log_cache_create(1, 2); + TEST_CHECK(worker.log_cache != NULL); + if (!worker.log_cache) { + return; + } + + previous_worker = flb_worker_get(); + FLB_TLS_SET(flb_worker_ctx, &worker); + + ret = flb_log_suppress_check(3, TEST_RECORD_01); + TEST_CHECK(ret == FLB_FALSE); + + entry = flb_log_cache_exists(worker.log_cache, + TEST_RECORD_01, TEST_RECORD_01_SIZE); + TEST_CHECK(entry != NULL); + if (entry) { + entry->timestamp = time(NULL) - 2; + } + + ret = flb_log_suppress_check(1, TEST_RECORD_02); + TEST_CHECK(ret == FLB_FALSE); + + entry = flb_log_cache_exists(worker.log_cache, + TEST_RECORD_02, TEST_RECORD_02_SIZE); + TEST_CHECK(entry != NULL); + if (entry) { + entry->timestamp = time(NULL) - 2; + } + + ret = flb_log_suppress_check(3, TEST_RECORD_03); + TEST_CHECK(ret == FLB_FALSE); + + entry = flb_log_cache_exists(worker.log_cache, + TEST_RECORD_03, TEST_RECORD_03_SIZE); + TEST_CHECK(entry != NULL); + + entry = flb_log_cache_exists(worker.log_cache, + TEST_RECORD_02, TEST_RECORD_02_SIZE); + TEST_CHECK(entry == NULL); + + ret = flb_log_suppress_check(3, TEST_RECORD_01); + TEST_CHECK(ret == FLB_TRUE); + + FLB_TLS_SET(flb_worker_ctx, previous_worker); + flb_log_cache_destroy(worker.log_cache); +} + TEST_LIST = { { "cache_basic_timeout" , cache_basic_timeout }, { "cache_one_slot" , cache_one_slot }, + { "suppress_interval_longer_than_cache_timeout", + suppress_interval_longer_than_cache_timeout }, + { "suppress_interval_shorter_than_cache_timeout", + suppress_interval_shorter_than_cache_timeout }, + { "suppress_interval_preserved_during_cache_replacement", + suppress_interval_preserved_during_cache_replacement }, { 0 } };