diff --git a/libs/visor_http_client/CMakeLists.txt b/libs/visor_http_client/CMakeLists.txt index 11184933a..4a3b5f16c 100644 --- a/libs/visor_http_client/CMakeLists.txt +++ b/libs/visor_http_client/CMakeLists.txt @@ -4,13 +4,14 @@ find_package(CURL REQUIRED) find_package(uvw REQUIRED) find_package(httplib REQUIRED) find_package(Catch2 REQUIRED) +find_package(nlohmann_json REQUIRED) add_library(VisorHttpClient STATIC HttpClient.cpp HttpCheck.cpp) # Namespaced alias for consistency with the other libs (Visor::Lib::Dns, Visor::Lib::Tcp, ...) # and cleaner downstream consumption. add_library(Visor::Lib::Http ALIAS VisorHttpClient) target_include_directories(VisorHttpClient PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(VisorHttpClient PUBLIC CURL::libcurl uvw::uvw) +target_link_libraries(VisorHttpClient PUBLIC CURL::libcurl uvw::uvw PRIVATE nlohmann_json::nlohmann_json) add_executable(unit-tests-visor-http-client test_http_client.cpp) target_link_libraries(unit-tests-visor-http-client PRIVATE VisorHttpClient Catch2::Catch2WithMain httplib::httplib) diff --git a/libs/visor_http_client/HttpCheck.cpp b/libs/visor_http_client/HttpCheck.cpp index 7dbdf478f..ffa54accb 100644 --- a/libs/visor_http_client/HttpCheck.cpp +++ b/libs/visor_http_client/HttpCheck.cpp @@ -1,5 +1,8 @@ #include "HttpCheck.h" -#include // curl_getdate (cpp only — the header stays curl-free) +#include +#include +#include // curl_getdate, CURL_HTTP_VERSION_* (cpp only — the header stays curl-free) +#include // JSON pointer parsing (cpp only — the header stays nlohmann-free) #include namespace visor::http { @@ -100,4 +103,166 @@ uint64_t parse_cert_expire_date(const std::string &date_str) time_t t = curl_getdate(date_str.c_str(), nullptr); return t > 0 ? static_cast(t) : 0; } + +JsonPointerCheck JsonPointerCheck::compile(const std::string &ptr, const std::string &equals, bool equals_set) +{ + JsonPointerCheck c; + try { + (void)nlohmann::json::json_pointer(ptr); // validate RFC 6901 syntax + } catch (const std::exception &) { + throw std::invalid_argument("json_path is not a valid JSON Pointer (RFC 6901): '" + ptr + "'"); + } + c._pointer = ptr; + c._has_expected = equals_set; + c._expected = equals; + c._configured = true; + return c; +} + +bool JsonPointerCheck::configured() const +{ + return _configured; +} + +bool JsonPointerCheck::matches(const std::string &body) const +{ + nlohmann::json doc = nlohmann::json::parse(body, nullptr, false); // no exceptions + if (doc.is_discarded()) { + return false; // not valid JSON + } + nlohmann::json::json_pointer p(_pointer); + try { + if (!doc.contains(p)) { + return false; // pointer does not resolve + } + if (!_has_expected) { + return true; // presence-only + } + const nlohmann::json &v = doc.at(p); + std::string actual = v.is_string() ? v.get() : v.dump(); // compact text for non-strings + return actual == _expected; + } catch (const nlohmann::json::exception &) { + // defensive: nlohmann 3.12's contains()/at() did not throw on a deeply-missing parent in + // observed testing, but guard against it anyway since it is not guaranteed by the API. + return false; + } +} + +BodyNegativeCheck BodyNegativeCheck::compile(const std::string ¬_substr, const std::string ¬_regex_pattern) +{ + BodyNegativeCheck c; + c.not_substring = not_substr; + if (!not_regex_pattern.empty()) { + try { + c.not_regex.emplace(not_regex_pattern, std::regex::ECMAScript); + } catch (const std::regex_error &) { + // never quote the pattern — it can embed secrets + throw std::invalid_argument("body_not_matches_regex is not a valid ECMAScript regular expression"); + } + } + return c; +} + +bool BodyNegativeCheck::matches(const std::string &body) const +{ + if (!not_substring.empty() && body.find(not_substring) != std::string::npos) { + return false; + } + if (not_regex.has_value() && std::regex_search(body, *not_regex)) { + return false; + } + return true; +} + +bool iequals_ascii(const std::string &a, const std::string &b) +{ + if (a.size() != b.size()) { + return false; + } + for (size_t i = 0; i < a.size(); ++i) { + if (std::tolower(static_cast(a[i])) != std::tolower(static_cast(b[i]))) { + return false; + } + } + return true; +} + +HeaderMatchers HeaderMatchers::compile(const std::vector> &fail_if_matches, + const std::vector> &fail_if_not_matches) +{ + HeaderMatchers m; + auto build = [](const std::vector> &src, std::vector &dst) { + for (const auto &[name, pat] : src) { + try { + dst.push_back(HeaderMatcher{name, std::regex(pat, std::regex::ECMAScript)}); + } catch (const std::regex_error &) { + // never quote the pattern — it can embed secrets; name the header instead + throw std::invalid_argument("header value_regex is not a valid ECMAScript regular expression (header '" + name + "')"); + } + } + }; + build(fail_if_matches, m._fail_if_matches); + build(fail_if_not_matches, m._fail_if_not_matches); + m._configured = !m._fail_if_matches.empty() || !m._fail_if_not_matches.empty(); + return m; +} + +bool HeaderMatchers::configured() const +{ + return _configured; +} + +bool HeaderMatchers::has_forbidden_rules() const +{ + return !_fail_if_matches.empty(); +} + +bool HeaderMatchers::matches(const std::vector> &headers) const +{ + for (const auto &hm : _fail_if_matches) { + for (const auto &[hn, hv] : headers) { + if (iequals_ascii(hn, hm.name) && std::regex_search(hv, hm.value_regex)) { + return false; // a forbidden header matched + } + } + } + for (const auto &hm : _fail_if_not_matches) { + bool any = false; + for (const auto &[hn, hv] : headers) { + if (iequals_ascii(hn, hm.name) && std::regex_search(hv, hm.value_regex)) { + any = true; + break; + } + } + if (!any) { + return false; // required header/value not present + } + } + return true; +} + +uint64_t parse_http_date(const std::string &date_str) +{ + if (date_str.empty()) { + return 0; + } + time_t t = curl_getdate(date_str.c_str(), nullptr); + return t > 0 ? static_cast(t) : 0; +} + +std::string http_version_name(long v) +{ + switch (v) { + case CURL_HTTP_VERSION_1_0: + return "1.0"; + case CURL_HTTP_VERSION_1_1: + return "1.1"; + case CURL_HTTP_VERSION_2_0: + return "2"; + case CURL_HTTP_VERSION_3: + return "3"; + default: + return ""; + } +} } diff --git a/libs/visor_http_client/HttpCheck.h b/libs/visor_http_client/HttpCheck.h index 9c0e3c51a..26f6f0078 100644 --- a/libs/visor_http_client/HttpCheck.h +++ b/libs/visor_http_client/HttpCheck.h @@ -3,6 +3,7 @@ #include #include #include +#include #include namespace visor::http { @@ -35,4 +36,68 @@ struct BodyCheck { // Parse a curl CERTINFO "Expire date:" value, e.g. "Aug 15 12:00:00 2026 GMT", to unix epoch. // Returns 0 on parse failure. (Pure string->epoch; the CERTINFO iteration lives in HttpClient.) uint64_t parse_cert_expire_date(const std::string &date_str); + +// RFC 6901 JSON Pointer assertion over a JSON body. +class JsonPointerCheck +{ +public: + JsonPointerCheck() = default; // not configured; configured()==false + // ptr: RFC 6901 pointer (validated; throws std::invalid_argument if malformed). + // equals_set=false => presence-only (pointer must resolve). equals_set=true => value's + // string form must equal `equals`. + static JsonPointerCheck compile(const std::string &ptr, const std::string &equals, bool equals_set); + bool configured() const; + bool matches(const std::string &body) const; // true = PASS +private: + std::string _pointer; + bool _has_expected{false}; + std::string _expected; + bool _configured{false}; +}; + +// Inverse body assertions: body must NOT contain `substring` and must NOT match `regex`. +struct BodyNegativeCheck { + std::string not_substring; // empty => not checked + std::optional not_regex; // nullopt => not checked + bool configured() const { return !not_substring.empty() || not_regex.has_value(); } + static BodyNegativeCheck compile(const std::string ¬_substr, const std::string ¬_regex_pattern); + bool matches(const std::string &body) const; // true = PASS (neither negative hit) +}; + +// Response-header assertions (fail_if_header_matches / fail_if_header_not_matches). +struct HeaderMatcher { + std::string name; // case-insensitive header name + std::regex value_regex; // compiled ECMAScript +}; +class HeaderMatchers +{ +public: + // Each pair is (name, value_regex_pattern). compile throws std::invalid_argument on a bad + // pattern (never quoting it). fail_if_matches: PASS unless some header `name` value matches. + // fail_if_not_matches: PASS only if some header `name` value matches. + static HeaderMatchers compile(const std::vector> &fail_if_matches, + const std::vector> &fail_if_not_matches); + bool configured() const; + // True when any fail_if_matches (forbidden-header) rule is configured. A forbidden-header PASS is + // only "no match seen in the captured headers", so it cannot be trusted when capture truncated; + // a required-header (fail_if_not_matches) PASS is a positive presence proof that truncation + // cannot invalidate. The probe uses this to decide whether to fail-safe on truncation. + bool has_forbidden_rules() const; + // headers: response headers as (name,value); name compared case-insensitively. + bool matches(const std::vector> &headers) const; // true = PASS +private: + std::vector _fail_if_matches; + std::vector _fail_if_not_matches; + bool _configured{false}; +}; + +// Parse an HTTP-date (Last-Modified) to unix epoch via curl_getdate; 0 on failure/empty. +uint64_t parse_http_date(const std::string &date_str); + +// Map CURLINFO_HTTP_VERSION (CURL_HTTP_VERSION_*) to "1.0"/"1.1"/"2"/"3"/"" (unknown). +std::string http_version_name(long curl_http_version); + +// Portable ASCII case-insensitive equality (no strcasecmp — MSVC). Used for header-name matching +// here and by the probe's Last-Modified lookup (Task 5). +bool iequals_ascii(const std::string &a, const std::string &b); } diff --git a/libs/visor_http_client/HttpClient.cpp b/libs/visor_http_client/HttpClient.cpp index 293fe9eee..733f827aa 100644 --- a/libs/visor_http_client/HttpClient.cpp +++ b/libs/visor_http_client/HttpClient.cpp @@ -81,6 +81,25 @@ std::optional validate_http_url(const std::string &url) return std::nullopt; } +std::string build_connect_to_entry(const std::string &resolve_entry) +{ + // Parse "host:port:address" (address may itself contain ':' when IPv6, so the address is + // everything after the SECOND colon — the two leading colons delimit host and port). + auto c1 = resolve_entry.find(':'); + auto c2 = (c1 == std::string::npos) ? std::string::npos : resolve_entry.find(':', c1 + 1); + if (c1 == std::string::npos || c2 == std::string::npos || c1 == 0 || c2 == c1 + 1 || c2 + 1 >= resolve_entry.size()) { + return {}; // malformed: missing a colon or an empty host/port/address field + } + std::string host = resolve_entry.substr(0, c1); + std::string port = resolve_entry.substr(c1 + 1, c2 - c1 - 1); + std::string address = resolve_entry.substr(c2 + 1); + // curl's --connect-to HOST2 needs an IPv6 literal bracketed; bracket it if the operator didn't. + if (address.front() != '[' && address.find(':') != std::string::npos) { + address = "[" + address + "]"; + } + return host + ":" + port + ":" + address + ":" + port; +} + HttpClient::HttpClient(std::shared_ptr loop) : _loop(std::move(loop)) { @@ -174,6 +193,51 @@ size_t HttpClient::write_capture(char *ptr, size_t size, size_t nmemb, void *use return n; // always consume so curl doesn't abort the transfer } +size_t HttpClient::header_capture(char *buffer, size_t size, size_t nitems, void *userdata) +{ + size_t n = size * nitems; + auto *ctx = static_cast(userdata); + if (ctx && ctx->collect_headers) { + constexpr size_t kMaxHeaderBytes = 64 * 1024; // byte cap only (DoS guard); no header-count cap + std::string line(buffer, n); + // curl fires this callback for EVERY response in the transfer (each redirect hop, and any + // proxy CONNECT response). A new response begins with a status line "HTTP/..." (no colon). + // Reset so only the FINAL response's headers survive — header/Last-Modified assertions must + // evaluate the final response, matching blackbox/cloudprober semantics. + if (line.rfind("HTTP/", 0) == 0) { + // Reset ALL per-response capture state (including the truncation flag) so a redirect hop + // with oversized headers does not fail the assertion on a small FINAL response. + ctx->resp_headers.clear(); + ctx->resp_headers_bytes = 0; + ctx->headers_truncated = false; + return n; + } + auto colon = line.find(':'); + if (colon != std::string::npos) { + if (ctx->resp_headers_bytes + n > kMaxHeaderBytes) { + // Dropping a header line: header-based assertions can no longer be fully verified. + // Flag it so the caller fails conservatively rather than matching a partial set. + ctx->headers_truncated = true; + } else { + std::string name = line.substr(0, colon); + std::string value = line.substr(colon + 1); + auto trim = [](std::string &s) { + size_t b = s.find_first_not_of(" \t\r\n"); + size_t e = s.find_last_not_of(" \t\r\n"); + s = (b == std::string::npos) ? std::string() : s.substr(b, e - b + 1); + }; + trim(name); + trim(value); + if (!name.empty()) { + ctx->resp_headers.emplace_back(std::move(name), std::move(value)); + ctx->resp_headers_bytes += n; + } + } + } + } + return n; // always consume +} + void HttpClient::request(const HttpRequest &req, ResultCallback on_done) { if (_closed || !_multi) { @@ -217,6 +281,15 @@ void HttpClient::request(const HttpRequest &req, ResultCallback on_done) if (!req.proxy.empty()) { curl_easy_setopt(easy, CURLOPT_PROXY, req.proxy.c_str()); ctx->proxy = req.proxy; // retained only to redact it (and any embedded credentials) from error_msg + } else { + // No proxy configured: explicitly disable libcurl's ambient environment proxies + // (http_proxy/https_proxy/all_proxy). Setting CURLOPT_PROXY to "" disables proxy use even + // when such an env var is set. A netprobe measures the DIRECT path to the target (or the + // explicitly-configured proxy); honoring an ambient proxy would make results depend on the + // deployment/CI environment and would defeat the per-target resolve/CONNECT_TO pin (an env + // proxy + CONNECT_TO switches curl to tunnel mode, asking the proxy to reach the pinned + // address instead of connecting locally). + curl_easy_setopt(easy, CURLOPT_PROXY, ""); } if (!req.ca_file.empty()) { curl_easy_setopt(easy, CURLOPT_CAINFO, req.ca_file.c_str()); @@ -230,6 +303,35 @@ void HttpClient::request(const HttpRequest &req, ResultCallback on_done) if (req.collect_cert_info) { curl_easy_setopt(easy, CURLOPT_CERTINFO, 1L); } + ctx->collect_headers = req.collect_headers; + if (req.collect_headers) { + curl_easy_setopt(easy, CURLOPT_HEADERFUNCTION, &HttpClient::header_capture); + curl_easy_setopt(easy, CURLOPT_HEADERDATA, ctx.get()); + curl_easy_setopt(easy, CURLOPT_SUPPRESS_CONNECT_HEADERS, 1L); // don't feed proxy CONNECT headers to the callback + } + if (req.ip_resolve) { + curl_easy_setopt(easy, CURLOPT_IPRESOLVE, req.ip_resolve); + } + if (!req.resolve.empty()) { + // Apply per-target address overrides via CURLOPT_CONNECT_TO, NOT CURLOPT_RESOLVE. RESOLVE + // entries (without a leading '+') are inserted PERMANENTLY into the DNS cache of the shared + // per-stream multi handle, so a later target for the same host:port that did NOT configure + // an override would still be sent to the pinned address — the override would leak across + // targets. CONNECT_TO is scoped to this easy handle and never touches the DNS cache. + // Each configured entry "host:port:address" becomes CONNECT_TO "host:port:address:port" + // (connect-to-port = the original port; SNI/Host/cert verification keep the original host). + // build_connect_to_entry brackets IPv6 literals and drops malformed entries. + for (const auto &e : req.resolve) { + std::string connect_to = build_connect_to_entry(e); + if (connect_to.empty()) { + continue; // malformed (validated upstream in the netprobe input; defensive here) + } + struct curl_slist *appended = curl_slist_append(ctx->connect_to_list, connect_to.c_str()); + if (!appended) { /* OOM: leave list intact; skip (best-effort) */ break; } + ctx->connect_to_list = appended; + } + if (ctx->connect_to_list) curl_easy_setopt(easy, CURLOPT_CONNECT_TO, ctx->connect_to_list); + } if (!req.body.empty()) { // COPYPOSTFIELDS copies the bytes (curl owns them); size set first => binary-safe. curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, static_cast(req.body.size())); @@ -439,6 +541,13 @@ void HttpClient::check_multi_info() if (ct) { result.content_type = ct; // raw header value; consumers compare case-insensitively } + long http_ver = 0; + curl_easy_getinfo(easy, CURLINFO_HTTP_VERSION, &http_ver); + result.http_version = http_ver; + if (it != _easy.end() && it->second->collect_headers) { + result.headers = std::move(it->second->resp_headers); + result.headers_truncated = it->second->headers_truncated; + } } else { result.transport_ok = false; result.curl_code = msg->data.result; diff --git a/libs/visor_http_client/HttpClient.h b/libs/visor_http_client/HttpClient.h index 1ab887a53..778057264 100644 --- a/libs/visor_http_client/HttpClient.h +++ b/libs/visor_http_client/HttpClient.h @@ -49,7 +49,16 @@ class HttpClient size_t capture_max{64 * 1024}; // cap on captured body bytes (from HttpRequest.capture_max_bytes) bool truncated{false}; // set by write_capture when the body exceeds capture_max std::string response; // captured body (bounded to capture_max) - ~EasyContext() { if (headers) curl_slist_free_all(headers); } + bool collect_headers{false}; + std::vector> resp_headers; + size_t resp_headers_bytes{0}; + bool headers_truncated{false}; // a response header line was dropped at the byte cap + curl_slist *connect_to_list{nullptr}; // owned; freed in dtor after curl_easy_cleanup + ~EasyContext() + { + if (headers) curl_slist_free_all(headers); + if (connect_to_list) curl_slist_free_all(connect_to_list); + } }; // per-socket context: a uvw poll handle curl watches (owned in _sockets below) struct SocketContext { @@ -61,6 +70,7 @@ class HttpClient static int timer_cb(CURLM *multi, long timeout_ms, void *userp); static size_t write_discard(char *ptr, size_t size, size_t nmemb, void *userdata); static size_t write_capture(char *ptr, size_t size, size_t nmemb, void *userdata); + static size_t header_capture(char *buffer, size_t size, size_t nitems, void *userdata); void on_socket_event(curl_socket_t sockfd, int events); void on_timeout(); void check_multi_info(); @@ -80,4 +90,12 @@ class HttpClient // callers don't reach into the curl API). Returns std::nullopt when valid; otherwise a short, // human-readable reason (e.g. "is not a valid http(s) URL: ''") the caller can surface. std::optional validate_http_url(const std::string &url); + +// Build a CURLOPT_CONNECT_TO entry from a "host:port:address" resolve override. curl's --connect-to +// format is HOST1:PORT1:HOST2:PORT2 and requires an IPv6 literal in HOST2 to be bracketed, so an +// unbracketed IPv6 address is normalized to "[addr]" here. PORT2 reuses the original port (the +// override only redirects the connection address; SNI/Host/cert verification keep the original +// host). Returns "" when the entry is malformed (missing either of the two required colons, or an +// empty host/port/address field) so the caller can skip it rather than emit a bad CONNECT_TO. +std::string build_connect_to_entry(const std::string &resolve_entry); } diff --git a/libs/visor_http_client/HttpTypes.h b/libs/visor_http_client/HttpTypes.h index 5f8a866a2..b9b92dd3c 100644 --- a/libs/visor_http_client/HttpTypes.h +++ b/libs/visor_http_client/HttpTypes.h @@ -1,6 +1,7 @@ #pragma once #include #include +#include #include namespace visor::http { @@ -28,6 +29,9 @@ struct HttpRequest { std::string cert_file; // CURLOPT_SSLCERT (client cert, empty => none) std::string key_file; // CURLOPT_SSLKEY (client key, empty => none) std::string user_agent; // CURLOPT_USERAGENT (empty => curl default) + bool collect_headers{false}; // when true, capture the FINAL response's headers into HttpResult.headers + long ip_resolve{0}; // CURLOPT_IPRESOLVE (0 => curl default/whatever; e.g. CURL_IPRESOLVE_V4/V6) + std::vector resolve; // per-target address overrides, each "host:port:address"; applied via CURLOPT_CONNECT_TO (per-handle, no shared DNS-cache leak) }; struct HttpResult { bool transport_ok{false}; @@ -40,6 +44,9 @@ struct HttpResult { std::string error_msg; // human-readable curl error detail when !transport_ok uint64_t cert_expiry_epoch{0}; // earliest "Expire date:" across the TLS chain when HttpRequest.collect_cert_info; 0 for plain http or on parse failure uint64_t response_size{0}; // CURLINFO_SIZE_DOWNLOAD_T; populated on every transport_ok, independent of capture_response + std::vector> headers; // FINAL response's headers when HttpRequest.collect_headers (redirect/proxy-CONNECT hops excluded) + bool headers_truncated{false}; // true when a header was dropped at the capture byte cap (headers is partial) + long http_version{0}; // CURLINFO_HTTP_VERSION (e.g. CURL_HTTP_VERSION_1_1/2_0); populated on every transport_ok }; struct HttpSample { uint16_t status{0}; diff --git a/libs/visor_http_client/test_http_check.cpp b/libs/visor_http_client/test_http_check.cpp index 5d84fdf03..3ecabce23 100644 --- a/libs/visor_http_client/test_http_check.cpp +++ b/libs/visor_http_client/test_http_check.cpp @@ -1,6 +1,8 @@ #include "HttpCheck.h" #include #include +#include +#include using namespace visor::http; @@ -63,3 +65,69 @@ TEST_CASE("parse_cert_expire_date", "[http][check]") CHECK(parse_cert_expire_date("not a date") == 0); CHECK(parse_cert_expire_date("") == 0); } + +TEST_CASE("JsonPointerCheck", "[http][check]") +{ + auto eq = JsonPointerCheck::compile("/data/status", "ok", true); + CHECK(eq.configured()); + CHECK(eq.matches(R"({"data":{"status":"ok"}})")); + CHECK_FALSE(eq.matches(R"({"data":{"status":"down"}})")); + CHECK_FALSE(eq.matches(R"({"data":{}})")); // pointer doesn't resolve + CHECK_FALSE(eq.matches("not json")); // parse failure + + // number/bool compared by compact JSON text + CHECK(JsonPointerCheck::compile("/code", "200", true).matches(R"({"code":200})")); + CHECK(JsonPointerCheck::compile("/ok", "true", true).matches(R"({"ok":true})")); + + // presence-only (no equals) + auto present = JsonPointerCheck::compile("/data/status", "", false); + CHECK(present.matches(R"({"data":{"status":"anything"}})")); + CHECK_FALSE(present.matches(R"({"data":{}})")); + + CHECK_FALSE(JsonPointerCheck{}.configured()); + CHECK_THROWS_AS(JsonPointerCheck::compile("data/status", "", false), std::invalid_argument); // no leading '/' +} + +TEST_CASE("BodyNegativeCheck", "[http][check]") +{ + auto n = BodyNegativeCheck::compile("traceback", "ERROR|FATAL"); + CHECK(n.configured()); + CHECK(n.matches("all good")); + CHECK_FALSE(n.matches("python traceback (most recent call last)")); + CHECK_FALSE(n.matches("status: FATAL")); + CHECK_FALSE(BodyNegativeCheck::compile("", "").configured()); + CHECK_THROWS_AS(BodyNegativeCheck::compile("", "(unclosed"), std::invalid_argument); +} + +TEST_CASE("HeaderMatchers", "[http][check]") +{ + std::vector> hdrs = { + {"Content-Type", "application/json"}, {"X-Cache", "HIT"}}; + // fail_if_matches: X-Debug present with any value -> here absent -> PASS + CHECK(HeaderMatchers::compile({{"X-Debug", ".+"}}, {}).matches(hdrs)); + // fail_if_matches: X-Cache matches HIT -> FAIL + CHECK_FALSE(HeaderMatchers::compile({{"X-Cache", "HIT"}}, {}).matches(hdrs)); + // fail_if_not_matches: Content-Type must match application/json -> present -> PASS + CHECK(HeaderMatchers::compile({}, {{"content-type", "application/json"}}).matches(hdrs)); // case-insensitive name + // fail_if_not_matches: requires an X-Missing match -> absent -> FAIL + CHECK_FALSE(HeaderMatchers::compile({}, {{"X-Missing", ".+"}}).matches(hdrs)); + CHECK_FALSE(HeaderMatchers::compile({}, {}).configured()); + CHECK_THROWS_AS(HeaderMatchers::compile({{"X", "(bad"}}, {}), std::invalid_argument); + // has_forbidden_rules(): true only when a fail_if_matches rule exists (drives the probe's + // truncation fail-safe — a required-only config must not fail-safe on truncation). + CHECK(HeaderMatchers::compile({{"X-Debug", ".+"}}, {}).has_forbidden_rules()); + CHECK(HeaderMatchers::compile({{"X-Debug", ".+"}}, {{"X-Ok", "1"}}).has_forbidden_rules()); + CHECK_FALSE(HeaderMatchers::compile({}, {{"X-Ok", "1"}}).has_forbidden_rules()); + CHECK_FALSE(HeaderMatchers::compile({}, {}).has_forbidden_rules()); +} + +TEST_CASE("parse_http_date + http_version_name", "[http][check]") +{ + CHECK(parse_http_date("Wed, 21 Oct 2026 07:28:00 GMT") != 0); + CHECK(parse_http_date("") == 0); + CHECK(parse_http_date("garbage") == 0); + CHECK(http_version_name(CURL_HTTP_VERSION_1_1) == "1.1"); + CHECK(http_version_name(CURL_HTTP_VERSION_2_0) == "2"); + CHECK(http_version_name(CURL_HTTP_VERSION_3) == "3"); + CHECK(http_version_name(0) == ""); +} diff --git a/libs/visor_http_client/test_http_client.cpp b/libs/visor_http_client/test_http_client.cpp index abd7ac088..2aec1eca9 100644 --- a/libs/visor_http_client/test_http_client.cpp +++ b/libs/visor_http_client/test_http_client.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include using namespace visor::http; @@ -400,6 +401,253 @@ TEST_CASE("HttpClient body capture cap + truncation flag", "[http][client]") if (server_thread.joinable()) server_thread.join(); } +TEST_CASE("HttpClient captures response headers + http_version", "[http][client]") +{ + httplib::Server svr; + svr.Get("/hdrs", [](const httplib::Request &, httplib::Response &res) { + res.set_header("X-Cache", "HIT"); + res.set_content("{\"ok\":true}", "application/json"); + }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread server_thread([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, server_thread}; + svr.wait_until_ready(); + + auto loop = uvw::loop::create(); + HttpClient client(loop); + std::vector results; + auto on_done = [&](const HttpResult &r) { results.push_back(r); }; + + HttpRequest req; + req.url = "http://127.0.0.1:" + std::to_string(port) + "/hdrs"; + req.collect_headers = true; + req.timeout_ms = 2000; + client.request(req, on_done); + auto wd = arm_watchdog(loop, 5000); + loop->run(); + disarm_watchdog(loop, wd); + + REQUIRE(results.size() == 1); + CHECK(results[0].transport_ok); + CHECK(results[0].http_version != 0); + bool found = false; + for (auto &[n, v] : results[0].headers) { + if (n == "X-Cache" && v == "HIT") found = true; + } + CHECK(found); + CHECK_FALSE(results[0].headers_truncated); // a handful of headers is well under the cap + + client.close(); + loop->run(); + svr.stop(); + if (server_thread.joinable()) server_thread.join(); +} + +TEST_CASE("HttpClient flags header truncation past the capture byte cap", "[http][client]") +{ + httplib::Server svr; + svr.Get("/many", [](const httplib::Request &, httplib::Response &res) { + // ~1400 headers of ~64 bytes each => ~90 KB, well past the 64 KB capture cap. + for (int i = 0; i < 1400; ++i) { + res.set_header("X-Pad-" + std::to_string(i), std::string(48, 'y')); + } + res.set_content("ok", "text/plain"); + }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread server_thread([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, server_thread}; + svr.wait_until_ready(); + + auto loop = uvw::loop::create(); + HttpClient client(loop); + std::vector results; + auto on_done = [&](const HttpResult &r) { results.push_back(r); }; + + HttpRequest req; + req.url = "http://127.0.0.1:" + std::to_string(port) + "/many"; + req.collect_headers = true; + req.timeout_ms = 3000; + client.request(req, on_done); + auto wd = arm_watchdog(loop, 6000); + loop->run(); + disarm_watchdog(loop, wd); + + REQUIRE(results.size() == 1); + CHECK(results[0].transport_ok); + CHECK(results[0].headers_truncated); // capture stopped at the byte cap; caller must fail-safe + + client.close(); + loop->run(); + svr.stop(); + if (server_thread.joinable()) server_thread.join(); +} + +TEST_CASE("HttpClient response header capture resets across a redirect hop", "[http][client]") +{ + // Proves the adversarial-review finding: CURLOPT_HEADERFUNCTION fires for EVERY response in + // the transfer (each redirect hop too). The intermediate (302) response carries a header that + // must NOT survive into the result; only the FINAL (200) response's headers should. + httplib::Server svr; + svr.Get("/redir", [](const httplib::Request &, httplib::Response &res) { + res.set_header("X-Hop", "intermediate"); + res.set_redirect("/final"); + }); + svr.Get("/final", [](const httplib::Request &, httplib::Response &res) { + res.set_header("X-Hop", "final"); + res.set_content("done", "text/plain"); + }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread server_thread([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, server_thread}; + svr.wait_until_ready(); + + auto loop = uvw::loop::create(); + HttpClient client(loop); + std::vector results; + auto on_done = [&](const HttpResult &r) { results.push_back(r); }; + + HttpRequest req; + req.url = "http://127.0.0.1:" + std::to_string(port) + "/redir"; + req.collect_headers = true; + req.follow_redirects = true; + req.timeout_ms = 2000; + client.request(req, on_done); + auto wd = arm_watchdog(loop, 5000); + loop->run(); + disarm_watchdog(loop, wd); + + REQUIRE(results.size() == 1); + CHECK(results[0].transport_ok); + CHECK(results[0].status_code == 200); // followed through to /final + int hop_count = 0; + std::string last_hop_value; + for (auto &[n, v] : results[0].headers) { + if (n == "X-Hop") { + ++hop_count; + last_hop_value = v; + } + } + CHECK(hop_count == 1); // the intermediate hop's X-Hop must have been cleared, not appended + CHECK(last_hop_value == "final"); + + client.close(); + loop->run(); + svr.stop(); + if (server_thread.joinable()) server_thread.join(); +} + +TEST_CASE("HttpClient clears header-truncation flag across a redirect hop", "[http][client]") +{ + // A redirect hop with oversized headers must NOT leave headers_truncated set when the FINAL + // response's headers fit under the cap — otherwise a header assertion that should pass on the + // final response is failed conservatively by the probe. + httplib::Server svr; + svr.Get("/redir", [](const httplib::Request &, httplib::Response &res) { + for (int i = 0; i < 1400; ++i) { // oversized intermediate headers => past the 64 KB cap + res.set_header("X-Pad-" + std::to_string(i), std::string(48, 'z')); + } + res.set_redirect("/final"); + }); + svr.Get("/final", [](const httplib::Request &, httplib::Response &res) { + res.set_header("X-Cache", "HIT"); // small final headers, comfortably under the cap + res.set_content("done", "text/plain"); + }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread server_thread([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, server_thread}; + svr.wait_until_ready(); + + auto loop = uvw::loop::create(); + HttpClient client(loop); + std::vector results; + auto on_done = [&](const HttpResult &r) { results.push_back(r); }; + + HttpRequest req; + req.url = "http://127.0.0.1:" + std::to_string(port) + "/redir"; + req.collect_headers = true; + req.follow_redirects = true; + req.timeout_ms = 3000; + client.request(req, on_done); + auto wd = arm_watchdog(loop, 6000); + loop->run(); + disarm_watchdog(loop, wd); + + REQUIRE(results.size() == 1); + CHECK(results[0].transport_ok); + CHECK(results[0].status_code == 200); + CHECK_FALSE(results[0].headers_truncated); // the flag from the oversized hop must have reset + bool found = false; + for (auto &[n, v] : results[0].headers) { + if (n == "X-Cache" && v == "HIT") found = true; + } + CHECK(found); + + client.close(); + loop->run(); + svr.stop(); + if (server_thread.joinable()) server_thread.join(); +} + +TEST_CASE("build_connect_to_entry brackets IPv6 and reuses the original port", "[http][client]") +{ + // IPv4: address is copied verbatim; the original port is appended as PORT2. + CHECK(build_connect_to_entry("example.com:443:1.2.3.4") == "example.com:443:1.2.3.4:443"); + // Unbracketed IPv6: the address must be bracketed for curl's HOST2 field. + CHECK(build_connect_to_entry("example.com:443:2001:db8::1") == "example.com:443:[2001:db8::1]:443"); + // Already-bracketed IPv6: left as-is (no double brackets). + CHECK(build_connect_to_entry("example.com:443:[2001:db8::1]") == "example.com:443:[2001:db8::1]:443"); + // IPv6 loopback (address begins immediately after the port colon). + CHECK(build_connect_to_entry("h:80:::1") == "h:80:[::1]:80"); + // Malformed entries yield "" so the caller skips them. + CHECK(build_connect_to_entry("host:443").empty()); // only one colon + CHECK(build_connect_to_entry("host:443:").empty()); // empty address + CHECK(build_connect_to_entry(":443:1.2.3.4").empty()); // empty host + CHECK(build_connect_to_entry("host::1.2.3.4").empty()); // empty port +} + +TEST_CASE("HttpClient CONNECT_TO override reaches an IPv6-loopback server", "[http][client]") +{ + // Bind on IPv6 loopback; skip where the environment has no ::1 (some CI containers). + httplib::Server svr; + svr.Get("/ok", [](const httplib::Request &, httplib::Response &res) { res.set_content("v6", "text/plain"); }); + int port = svr.bind_to_any_port("::1"); + if (port <= 0) { + SKIP("no IPv6 loopback available"); + } + std::thread server_thread([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, server_thread}; + svr.wait_until_ready(); + + auto loop = uvw::loop::create(); + HttpClient client(loop); + std::vector results; + auto on_done = [&](const HttpResult &r) { results.push_back(r); }; + + HttpRequest req; + // The URL host never resolves; the UNBRACKETED IPv6 resolve override must pin the connection + // to [::1]:port. Proves build_connect_to_entry's bracketing makes curl accept the entry. + req.url = "http://pin.invalid:" + std::to_string(port) + "/ok"; + req.resolve = {"pin.invalid:" + std::to_string(port) + ":::1"}; + req.timeout_ms = 3000; + client.request(req, on_done); + auto wd = arm_watchdog(loop, 6000); + loop->run(); + disarm_watchdog(loop, wd); + + REQUIRE(results.size() == 1); + CHECK(results[0].transport_ok); + CHECK(results[0].status_code == 200); + + client.close(); + loop->run(); + svr.stop(); + if (server_thread.joinable()) server_thread.join(); +} + TEST_CASE("HttpClient redacts proxy credentials from transport error_msg", "[http][client]") { auto loop = uvw::loop::create(); @@ -427,3 +675,80 @@ TEST_CASE("HttpClient redacts proxy credentials from transport error_msg", "[htt client.close(); loop->run(); } + +#if !defined(_WIN32) +// RAII snapshot/restore of a set of environment variables. Restores on destruction — including when +// a Catch2 REQUIRE throws — so a test can mutate proxy env vars hermetically without leaking state +// into later tests. +namespace { +class ScopedEnv +{ +public: + explicit ScopedEnv(std::vector names) + : _names(std::move(names)) + { + for (const auto &n : _names) { + const char *v = ::getenv(n.c_str()); + _saved.emplace_back(v != nullptr, v ? std::string(v) : std::string()); + ::unsetenv(n.c_str()); // start from a known-clean slate + } + } + void set(const char *name, const char *value) { ::setenv(name, value, 1); } + ~ScopedEnv() + { + for (size_t i = 0; i < _names.size(); ++i) { + if (_saved[i].first) { + ::setenv(_names[i].c_str(), _saved[i].second.c_str(), 1); + } else { + ::unsetenv(_names[i].c_str()); + } + } + } + ScopedEnv(const ScopedEnv &) = delete; + ScopedEnv &operator=(const ScopedEnv &) = delete; + +private: + std::vector _names; + std::vector> _saved; +}; +} // namespace + +TEST_CASE("HttpClient ignores ambient environment proxies when none is configured", "[http][client]") +{ + // A netprobe with no configured proxy must connect DIRECTLY, not via an ambient env proxy. + // Point http_proxy at a port where nothing listens: if curl honored the env proxy the request + // would fail (connection refused to the proxy); because request() sets CURLOPT_PROXY="" for + // no-proxy requests, env proxies are disabled and the direct request to the local server + // succeeds. + // + // Hermeticity: curl also honors no_proxy/all_proxy, so the test clears EVERY proxy-related env + // var first (an ambient no_proxy listing 127.0.0.1 would otherwise bypass the proxy and make the + // test pass even with the fix reverted — a wrong-reason pass). ScopedEnv restores them after. + ScopedEnv env{{"http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", + "all_proxy", "ALL_PROXY", "no_proxy", "NO_PROXY"}}; + env.set("http_proxy", "http://127.0.0.1:1"); // port 1: nothing listens + + httplib::Server svr; + std::thread server_thread; + int port = start_test_server(svr, server_thread); + ServerGuard guard{svr, server_thread}; + + auto loop = uvw::loop::create(); + HttpClient client(loop); + std::vector results; + HttpRequest req; + req.url = "http://127.0.0.1:" + std::to_string(port) + "/ok"; // no req.proxy + req.timeout_ms = 3000; + client.request(req, [&](const HttpResult &r) { results.push_back(r); }); + auto wd = arm_watchdog(loop, 6000); + loop->run(); + disarm_watchdog(loop, wd); + + REQUIRE(results.size() == 1); + CHECK(results[0].transport_ok); // direct connection succeeded => ambient proxy was NOT used + CHECK(results[0].status_code == 200); + + client.close(); + loop->run(); +} +#endif diff --git a/src/handlers/netprobe/NetProbeStreamHandler.h b/src/handlers/netprobe/NetProbeStreamHandler.h index 889d5bca2..5bbf4e7cf 100644 --- a/src/handlers/netprobe/NetProbeStreamHandler.h +++ b/src/handlers/netprobe/NetProbeStreamHandler.h @@ -79,7 +79,7 @@ struct Target { , dns_failures(NET_PROBE_SCHEMA, {"dns_lookup_failures"}, "Total Net Probe failures when performing a DNS lookup") , timed_out(NET_PROBE_SCHEMA, {"packets_timeout"}, "Total Net Probe timeout transactions") , http_status_failures(NET_PROBE_SCHEMA, {"http_status_failures"}, "Total HTTP/DoH responses whose HTTP status failed the configured status checks (default: any status outside 2xx/3xx)") - , content_failures(NET_PROBE_SCHEMA, {"content_failures"}, "Total HTTP responses whose status passed but response-body checks failed") + , content_failures(NET_PROBE_SCHEMA, {"content_failures"}, "Total HTTP responses whose status passed but a response assertion (body/json/size/header/version) failed") , top_status_codes(NET_PROBE_SCHEMA, "status_code", {"top_status_codes"}, "Top HTTP status codes") , dns_response_failures(NET_PROBE_SCHEMA, {"dns_response_failures"}, "Total DoH responses with a success HTTP status (2xx/3xx) but a non-NOERROR or unparseable DNS response") , top_rcodes(NET_PROBE_SCHEMA, "rcode", {"top_rcodes"}, "Top DNS response codes observed") diff --git a/src/handlers/netprobe/README.md b/src/handlers/netprobe/README.md index b8678a000..0de224719 100644 --- a/src/handlers/netprobe/README.md +++ b/src/handlers/netprobe/README.md @@ -40,10 +40,23 @@ Unlike ping/tcp, HTTP targets are specified as full URLs. | `body_check_max_bytes` | uint64 | 524288 (512 KiB) | Maximum number of response-body bytes captured for `expected_body`/`expected_body_regex` evaluation. If a response body exceeds this, it is truncated and the **body check is skipped** for that sample (classified on status alone, with a warning logged) rather than risk a false `content_failures` on a match that lives past the cap, or an anchored regex matching the truncation boundary. Raise it if your health endpoints return large bodies whose match text is deep in the response. | | `body` | string | *(unset)* | Request body to send. Only valid when `http_method` is `POST`, `PUT`, or `PATCH` — set on any other method throws a config error at start. Like custom headers, a body-bearing probe does **not** follow redirects (a `307`/`308` preserves the method and body, which would resend the payload to the redirect target). | | `targets..headers` | map | *(unset)* | Per-target HTTP headers (name → value), e.g. `Authorization: Bearer `. Sent only on requests to that target. **Redaction note:** header *values* are never echoed back by `info_json`/status output — they're replaced with ``. Header *names* (e.g. `Authorization`) are still surfaced (as `header_names`) since they're useful for debugging and carry no secret. **Redirects:** when a target has custom headers, the probe does **not** follow HTTP redirects — libcurl would otherwise re-send the headers to the redirect target (possibly another host), leaking secret headers. A `30x` is then reported as the response status (configure `expected_status` if a redirect should count as success). | -| `proxy` | string | *(unset)* | HTTP/HTTPS proxy URL for all targets, e.g. `http://user:pass@proxy.example:3128`. Applies to `http` and `doh` only (not `tcp`/`ping`). Like headers, the configured value is fully redacted (``) wherever config is echoed back — even the host/port, since the whole string can carry embedded credentials. | +| `proxy` | string | *(unset)* | HTTP/HTTPS proxy URL for all targets, e.g. `http://user:pass@proxy.example:3128`. Applies to `http` and `doh` only (not `tcp`/`ping`). Like headers, the configured value is fully redacted (``) wherever config is echoed back — even the host/port, since the whole string can carry embedded credentials. When unset the probe connects **directly** and ignores ambient `http_proxy`/`https_proxy` environment variables, so results do not depend on the deployment environment. | | `tls.verify` | bool | `true` | Whether to verify the target's TLS certificate/hostname. Set `false` only for testing against self-signed endpoints. | | `tls.ca_file` | string | *(unset)* | Path to a CA bundle to trust in addition to (or instead of) the system store. Must exist at start time or the stream fails to start. | | `tls.cert_file` / `tls.key_file` | string | *(unset)* | Client certificate + private key for mutual TLS (mTLS). Must be set together — setting one without the other is a config error. | +| `json_path` | string | *(unset)* | RFC 6901 JSON Pointer (e.g. `/data/status`) evaluated against the response body parsed as JSON. With `json_equals`, the value at the pointer must equal it; alone, the pointer must simply resolve. Non-JSON body, unresolved pointer, or value mismatch → `content_failures`. | +| `json_equals` | string | *(unset)* | Expected value at `json_path`, compared as a string (numbers/bools compared against their compact JSON text, so `"200"` matches `200`). Requires `json_path`. | +| `not_contains` | string | *(unset)* | The response body must **not** contain this substring (inverse of `expected_body`). A hit → `content_failures`. | +| `body_not_matches_regex` | string | *(unset)* | The response body must **not** match this ECMAScript regex (inverse of `expected_body_regex`). A match → `content_failures`. | +| `min_response_size_bytes` / `max_response_size_bytes` | uint64 | *(unset)* | Bound the response body size (true downloaded size, exact even if the captured body was truncated). Out of bounds → `content_failures`. If both set, `min ≤ max`. | +| `fail_if_header_matches` | map | *(unset)* | Map of `header-name: value_regex`. Fail (`content_failures`) if a response header with that name (case-insensitive) has a value matching the regex. | +| `fail_if_header_not_matches` | map | *(unset)* | Map of `header-name: value_regex`. Fail unless some response header with that name matches the regex (i.e. requires the match). | +| `max_last_modified_diff_secs` | uint64 | *(unset)* | Fail if the response `Last-Modified` is older than this many seconds, or is absent/unparseable. | +| `valid_http_versions` | list of strings | *(unset)* | Allowed negotiated HTTP versions (`"1.0"`, `"1.1"`, `"2"`, `"3"`). A negotiated version outside the set → `content_failures`. | +| `targets..ip_version` | uint64 (4 or 6) | *(unset)* | Force IPv4 or IPv6 resolution for that target (`CURLOPT_IPRESOLVE`). Per-target, same key ping/tcp already accept. | +| `targets..resolve` | list of strings | *(unset)* | Per-target `host:port:address` overrides — pin the connect address without changing the `Host`/SNI/cert verification. Applied via `CURLOPT_CONNECT_TO` (scoped to that target's request; it does not populate the shared DNS cache, so an override on one target never leaks to another target for the same `host:port`). IPv6 literals may be written with or without brackets (`host:443:2001:db8::1` or `host:443:[2001:db8::1]`). | + +All of `json_path`/`json_equals`/`not_contains`/`body_not_matches_regex`/`min|max_response_size_bytes`/`fail_if_header_matches`/`fail_if_header_not_matches`/`max_last_modified_diff_secs`/`valid_http_versions` are **response assertions**: they run only after the status check passes, are ANDed together, and a failure of any is counted as `content_failures` (never `successes`, never `http_status_failures`). Body-reading assertions (`expected_body`/`expected_body_regex`/`not_contains`/`body_not_matches_regex`/`json_path`) are **skipped** (with a warning) when the response body exceeded `body_check_max_bytes`; size/header/version assertions are unaffected. Assertion string values (`not_contains`, `body_not_matches_regex`, `json_equals`, and the header `value_regex`s) are redacted from `info_json` like other secrets. #### Success semantics diff --git a/src/inputs/netprobe/DohProbe.cpp b/src/inputs/netprobe/DohProbe.cpp index 697b4ddff..72ab27ef3 100644 --- a/src/inputs/netprobe/DohProbe.cpp +++ b/src/inputs/netprobe/DohProbe.cpp @@ -147,6 +147,8 @@ bool DohProbe::start(std::shared_ptr io_loop) req.user_agent = _opts.user_agent; req.verify_tls = _opts.tls_verify; req.collect_cert_info = true; + req.ip_resolve = _ip_resolve; + req.resolve = _resolve; const std::string name = _name; const std::string qname = _wire_qname; // "" for the root; matches what pcpp getName() returns const uint16_t qtype_code = _qtype_code; diff --git a/src/inputs/netprobe/DohProbe.h b/src/inputs/netprobe/DohProbe.h index ac9c2a601..f6a6b0306 100644 --- a/src/inputs/netprobe/DohProbe.h +++ b/src/inputs/netprobe/DohProbe.h @@ -23,6 +23,8 @@ class DohProbe final : public NetProbe std::string _qtype; // e.g. "A" std::shared_ptr _client; HttpProbeOptions _opts; // uses only proxy/tls/user_agent + collect_cert_info + long _ip_resolve{0}; // CURLOPT_IPRESOLVE override (0 => curl default); per-target ip_version + std::vector _resolve; // CURLOPT_RESOLVE entries, each "host:port:address"; per-target resolve DohResultCallback _doh_result; std::shared_ptr _interval_timer; std::string _query_wire; // pre-built DNS query (wire format), built in start() @@ -36,7 +38,8 @@ class DohProbe final : public NetProbe public: DohProbe(uint16_t id, const std::string &name, std::string url, std::string method, std::string qname, std::string qtype, - std::shared_ptr client, HttpProbeOptions opts, DohResultCallback doh_result) + std::shared_ptr client, HttpProbeOptions opts, + long ip_resolve, std::vector resolve, DohResultCallback doh_result) : NetProbe(id, name, pcpp::IPAddress(), std::string()) , _url(std::move(url)) , _method(std::move(method)) @@ -44,6 +47,8 @@ class DohProbe final : public NetProbe , _qtype(std::move(qtype)) , _client(std::move(client)) , _opts(std::move(opts)) + , _ip_resolve(ip_resolve) + , _resolve(std::move(resolve)) , _doh_result(std::move(doh_result)) {} ~DohProbe() = default; bool start(std::shared_ptr io_loop) override; diff --git a/src/inputs/netprobe/HttpProbe.cpp b/src/inputs/netprobe/HttpProbe.cpp index 16a9cb7b8..f99baf981 100644 --- a/src/inputs/netprobe/HttpProbe.cpp +++ b/src/inputs/netprobe/HttpProbe.cpp @@ -46,8 +46,11 @@ bool HttpProbe::start(std::shared_ptr io_loop) req.user_agent = _opts.user_agent; req.verify_tls = _opts.tls_verify; req.collect_cert_info = true; - req.capture_response = _opts.body_check.configured(); + req.capture_response = _opts.body_check.configured() || _opts.json_check.configured() || _opts.body_negative.configured(); req.capture_max_bytes = _opts.body_check_max_bytes; + req.collect_headers = _opts.header_matchers.configured() || _opts.max_last_modified_diff > 0; + req.ip_resolve = _ip_resolve; + req.resolve = _resolve; const std::string name = _name; auto http_result = _http_result; auto fail = _fail; @@ -69,17 +72,100 @@ bool HttpProbe::start(std::shared_ptr io_loop) } s.status_ok = status_ok; s.content_check = 0; - if (status_ok && opts.body_check.configured()) { - if (r.body_truncated) { - // The captured body is only a prefix (it exceeded body_check_max_bytes): a - // match beyond the cap would be missed, and an anchored regex could match the - // artificial truncation boundary. We can't authoritatively evaluate the body, - // so classify on status alone (content_check stays NotChecked) and warn. - if (auto logger = spdlog::get("visor")) { - logger->warn("netprobe http[{}]: response body exceeded the {}-byte capture limit; body check skipped (raise body_check_max_bytes)", name, opts.body_check_max_bytes); + if (status_ok) { + bool checked = false, pass = true; + // Body-based assertions: skipped (inconclusive) if the body was truncated. A + // partial body can't authoritatively pass/fail a substring/regex/JSON/negative + // check — a match beyond the cap would be missed, and an anchored regex could + // match the artificial truncation boundary. + bool body_assertions = opts.body_check.configured() || opts.json_check.configured() || opts.body_negative.configured(); + if (body_assertions) { + if (r.body_truncated) { + // A forbidden substring VISIBLE in the captured prefix is a definitive + // failure even on a truncated body — substring presence does not depend on + // the unseen tail. The other body assertions (positive matches, regex, and + // JSON parsing) stay inconclusive on a partial body and are skipped. + if (opts.body_negative.configured() && !opts.body_negative.not_substring.empty() + && r.response_body.find(opts.body_negative.not_substring) != std::string::npos) { + checked = true; + pass = false; + } else if (auto logger = spdlog::get("visor")) { + logger->warn("netprobe http[{}]: response body exceeded the {}-byte capture limit; body assertions skipped (raise body_check_max_bytes)", name, opts.body_check_max_bytes); + } + } else { + checked = true; + if (opts.body_check.configured() && !opts.body_check.matches(r.response_body)) { + pass = false; + } + if (pass && opts.json_check.configured() && !opts.json_check.matches(r.response_body)) { + pass = false; + } + if (pass && opts.body_negative.configured() && !opts.body_negative.matches(r.response_body)) { + pass = false; + } } - } else { - s.content_check = opts.body_check.matches(r.response_body) ? 1 : 2; + } + // Size bounds (use the true downloaded size, always exact — unaffected by capture + // truncation). + if (opts.min_response_size || opts.max_response_size) { + checked = true; + if (opts.min_response_size && r.response_size < *opts.min_response_size) { + pass = false; + } + if (pass && opts.max_response_size && r.response_size > *opts.max_response_size) { + pass = false; + } + } + // Response-header matchers. + if (pass && opts.header_matchers.configured()) { + checked = true; + if (!opts.header_matchers.matches(r.headers)) { + pass = false; + } else if (r.headers_truncated && opts.header_matchers.has_forbidden_rules()) { + // Headers were dropped at the capture cap. A forbidden-header (fail_if_ + // header_matches) PASS only means "no forbidden header was SEEN" — one + // could be among the dropped headers — so it can't be trusted; fail + // conservatively. A required-header (fail_if_header_not_matches) PASS is a + // positive presence proof that dropped headers cannot invalidate, so it is + // NOT failed here (guarded by has_forbidden_rules()). + pass = false; + if (auto logger = spdlog::get("visor")) { + logger->warn("netprobe http[{}]: response headers exceeded the capture limit; fail_if_header_matches assertion failed conservatively", name); + } + } + } + // Last-Modified freshness. + if (pass && opts.max_last_modified_diff > 0) { + checked = true; + uint64_t lm = 0; + for (const auto &[hn, hv] : r.headers) { + if (visor::http::iequals_ascii(hn, "Last-Modified")) { + lm = visor::http::parse_http_date(hv); + break; + } + } + uint64_t now = static_cast(stamp.tv_sec); + if (lm == 0 || (now > lm && now - lm > opts.max_last_modified_diff)) { + pass = false; + } + } + // Negotiated HTTP version. + if (pass && !opts.valid_http_versions.empty()) { + checked = true; + std::string ver = visor::http::http_version_name(r.http_version); + bool ok = false; + for (const auto &allowed : opts.valid_http_versions) { + if (allowed == ver) { + ok = true; + break; + } + } + if (!ok) { + pass = false; + } + } + if (checked) { + s.content_check = pass ? 1 : 2; } } // CERTINFO is only filled on transfers that performed a TLS handshake; reused diff --git a/src/inputs/netprobe/HttpProbe.h b/src/inputs/netprobe/HttpProbe.h index 96cda61b9..f7c1a9625 100644 --- a/src/inputs/netprobe/HttpProbe.h +++ b/src/inputs/netprobe/HttpProbe.h @@ -21,6 +21,8 @@ class HttpProbe final : public NetProbe std::shared_ptr _client; HttpProbeOptions _opts; std::vector _headers; + long _ip_resolve{0}; // CURLOPT_IPRESOLVE override (0 => curl default); per-target ip_version + std::vector _resolve; // CURLOPT_RESOLVE entries, each "host:port:address"; per-target resolve HttpResultCallback _http_result; std::shared_ptr _interval_timer; // CERTINFO is only populated on transfers that perform a TLS handshake; pooled-connection @@ -33,6 +35,7 @@ class HttpProbe final : public NetProbe public: HttpProbe(uint16_t id, const std::string &name, std::string url, std::string method, std::shared_ptr client, HttpProbeOptions opts, std::vector headers, + long ip_resolve, std::vector resolve, HttpResultCallback http_result) : NetProbe(id, name, pcpp::IPAddress(), std::string()) , _url(std::move(url)) @@ -40,6 +43,8 @@ class HttpProbe final : public NetProbe , _client(std::move(client)) , _opts(std::move(opts)) , _headers(std::move(headers)) + , _ip_resolve(ip_resolve) + , _resolve(std::move(resolve)) , _http_result(std::move(http_result)) {} ~HttpProbe() = default; bool start(std::shared_ptr io_loop) override; diff --git a/src/inputs/netprobe/HttpProbeOptions.h b/src/inputs/netprobe/HttpProbeOptions.h index c2bbd6126..d2509e1c7 100644 --- a/src/inputs/netprobe/HttpProbeOptions.h +++ b/src/inputs/netprobe/HttpProbeOptions.h @@ -4,7 +4,10 @@ #pragma once #include "HttpCheck.h" +#include +#include #include +#include namespace visor::input::netprobe { @@ -21,5 +24,15 @@ struct HttpProbeOptions { bool tls_verify{true}; std::string ca_file, cert_file, key_file; std::string user_agent; // "pktvisor/" VISOR_VERSION_NUM + + // v3: response-assertion config. Parsed/validated by NetProbeInputStream::start(); not yet + // evaluated by the probes (Task 5). + visor::http::JsonPointerCheck json_check; + visor::http::BodyNegativeCheck body_negative; + visor::http::HeaderMatchers header_matchers; + std::optional min_response_size; // set => lower bound (bytes) + std::optional max_response_size; // set => upper bound (bytes); 0 = require an empty body + uint64_t max_last_modified_diff{0}; // seconds; 0 = not checked + std::vector valid_http_versions; // empty = any }; } diff --git a/src/inputs/netprobe/NetProbeInputStream.cpp b/src/inputs/netprobe/NetProbeInputStream.cpp index f13282b1f..4330a1c08 100644 --- a/src/inputs/netprobe/NetProbeInputStream.cpp +++ b/src/inputs/netprobe/NetProbeInputStream.cpp @@ -13,6 +13,7 @@ #include "dns.h" #include "visor_config.h" #include +#include #include #include #ifdef __GNUC__ @@ -246,8 +247,107 @@ void NetProbeInputStream::start() _http_opts.key_file = tls->config_get("key_file"); } } + // ---- v3: stream-level response-assertion config (http only; parsed here, evaluated by the + // probe in a later task) ---- + { + std::string jp = config_exists("json_path") ? scalar_config_to_string(*this, "json_path", "config") : ""; + bool has_eq = config_exists("json_equals"); + std::string je = has_eq ? scalar_config_to_string(*this, "json_equals", "config") : ""; + if (has_eq && jp.empty()) { + throw NetProbeException("netprobe: 'json_equals' requires 'json_path'"); + } + if (!jp.empty()) { + try { + _http_opts.json_check = visor::http::JsonPointerCheck::compile(jp, je, has_eq); + } catch (const std::invalid_argument &e) { + throw NetProbeException(fmt::format("netprobe: {}", e.what())); + } + } + } + { + std::string ns = config_exists("not_contains") ? scalar_config_to_string(*this, "not_contains", "config") : ""; + std::string nr = config_exists("body_not_matches_regex") ? scalar_config_to_string(*this, "body_not_matches_regex", "config") : ""; + try { + _http_opts.body_negative = visor::http::BodyNegativeCheck::compile(ns, nr); + } catch (const std::invalid_argument &e) { + throw NetProbeException(fmt::format("netprobe: {}", e.what())); + } + } + if (config_exists("min_response_size_bytes")) { + _http_opts.min_response_size = config_get("min_response_size_bytes"); + } + if (config_exists("max_response_size_bytes")) { + _http_opts.max_response_size = config_get("max_response_size_bytes"); + } + if (_http_opts.min_response_size && _http_opts.max_response_size + && *_http_opts.min_response_size > *_http_opts.max_response_size) { + throw NetProbeException("netprobe: min_response_size_bytes must not exceed max_response_size_bytes"); + } + if (config_exists("max_last_modified_diff_secs")) { + _http_opts.max_last_modified_diff = config_get("max_last_modified_diff_secs"); + } + if (config_exists("valid_http_versions")) { + for (const auto &v : config_get("valid_http_versions")) { + if (v != "1.0" && v != "1.1" && v != "2" && v != "3") { + throw NetProbeException(fmt::format("netprobe: invalid valid_http_versions entry '{}' (use 1.0, 1.1, 2, or 3)", v)); + } + _http_opts.valid_http_versions.push_back(v); + } + } + // header matchers: each is a `header-name: value_regex` MAP read as a sub-Configurable, exactly + // like the per-target `headers` map (Configurable cannot load a YAML sequence-of-maps). Values + // are read with the typed-scalar reader so a numeric-looking regex is accepted. + { + std::vector> fm, fnm; + auto read_matchers = [this](const char *key, std::vector> &out) { + if (!config_exists(key)) return; + auto m = config_get>(key); + for (const auto &hname : m->get_all_keys()) { + out.emplace_back(hname, scalar_config_to_string(*m, hname, "value_regex")); + } + }; + read_matchers("fail_if_header_matches", fm); + read_matchers("fail_if_header_not_matches", fnm); + try { + _http_opts.header_matchers = visor::http::HeaderMatchers::compile(fm, fnm); + } catch (const std::invalid_argument &e) { + throw NetProbeException(fmt::format("netprobe: {}", e.what())); + } + } + _http_opts.user_agent = std::string("pktvisor/") + VISOR_VERSION_NUM; + // Per-target ip_version/resolve overrides (CURLOPT_IPRESOLVE/CURLOPT_RESOLVE), used for both + // http and doh targets. Mirrors the ping/tcp ip_version handling above, but scoped to a single + // target rather than the whole stream, and stored by target name for the http/doh build loop. + auto parse_target_ip_resolve = [this](const std::shared_ptr &config, const std::string &key) { + if (config->config_exists("ip_version")) { + auto v = config->config_get("ip_version"); + if (v != 4 && v != 6) { + throw NetProbeException("ip_version must be 4 or 6"); + } + _http_target_ipresolve[key] = (v == 6) ? CURL_IPRESOLVE_V6 : CURL_IPRESOLVE_V4; + } + if (config->config_exists("resolve")) { + std::vector entries; + for (const auto &e : config->config_get("resolve")) { + // host:port:address — require at least two ':' and a numeric port + auto c1 = e.find(':'); + auto c2 = (c1 == std::string::npos) ? std::string::npos : e.find(':', c1 + 1); + bool ok = c1 != std::string::npos && c2 != std::string::npos && c1 > 0 && c2 > c1 + 1 && c2 + 1 < e.size(); + if (ok) { + std::string port = e.substr(c1 + 1, c2 - c1 - 1); + ok = !port.empty() && port.find_first_not_of("0123456789") == std::string::npos; + } + if (!ok) { + throw NetProbeException(fmt::format("netprobe: target '{}' has an invalid resolve entry '{}' (expected host:port:address)", key, e)); + } + entries.push_back(e); + } + _http_target_resolve[key] = std::move(entries); + } + }; + if (!config_exists("targets")) { throw NetProbeException("no targets specified"); } else { @@ -276,6 +376,7 @@ void NetProbeInputStream::start() _http_target_headers[key] = std::move(joined); _http_target_header_names[key] = std::move(names); } + parse_target_ip_resolve(config, key); continue; } if (_type == TestType::DOH) { @@ -287,6 +388,7 @@ void NetProbeInputStream::start() throw NetProbeException("per-target 'headers' is not supported for test_type 'doh'"); } _doh_targets[key] = url; + parse_target_ip_resolve(config, key); continue; } uint32_t port{0}; @@ -359,7 +461,11 @@ void NetProbeInputStream::start() // http-only keys: check each individually so the thrown message names the offending key. { static const std::vector http_only_keys = { - "expected_status", "failure_status", "expected_body", "expected_body_regex", "body", "body_check_max_bytes"}; + "expected_status", "failure_status", "expected_body", "expected_body_regex", "body", "body_check_max_bytes", + "json_path", "json_equals", "not_contains", "body_not_matches_regex", + "min_response_size_bytes", "max_response_size_bytes", + "fail_if_header_matches", "fail_if_header_not_matches", + "max_last_modified_diff_secs", "valid_http_versions"}; if (_type != TestType::HTTP) { for (const auto &key : http_only_keys) { if (config_exists(key)) { @@ -561,7 +667,15 @@ void NetProbeInputStream::_create_netprobe_loop() if (auto it = _http_target_headers.find(key); it != _http_target_headers.end()) { headers = it->second; } - auto probe = std::make_unique(_id, key, url, _http_method, _http_client, _http_opts, headers, + long ip_resolve = 0; + if (auto it = _http_target_ipresolve.find(key); it != _http_target_ipresolve.end()) { + ip_resolve = it->second; + } + std::vector resolve; + if (auto it = _http_target_resolve.find(key); it != _http_target_resolve.end()) { + resolve = it->second; + } + auto probe = std::make_unique(_id, key, url, _http_method, _http_client, _http_opts, headers, ip_resolve, resolve, [this](visor::http::HttpSample sample, const std::string &name, timespec stamp) { _http_result_cb(sample, name, stamp); }); ++_id; probe->set_configs(_interval_msec, _timeout_msec, _packets_per_test, _packets_interval_msec, _packet_payload_size); @@ -573,7 +687,15 @@ void NetProbeInputStream::_create_netprobe_loop() } for (const auto &[key, url] : _doh_targets) { - auto probe = std::make_unique(_id, key, url, _doh_method, _doh_qname, _doh_qtype, _http_client, _http_opts, + long ip_resolve = 0; + if (auto it = _http_target_ipresolve.find(key); it != _http_target_ipresolve.end()) { + ip_resolve = it->second; + } + std::vector resolve; + if (auto it = _http_target_resolve.find(key); it != _http_target_resolve.end()) { + resolve = it->second; + } + auto probe = std::make_unique(_id, key, url, _doh_method, _doh_qname, _doh_qtype, _http_client, _http_opts, ip_resolve, resolve, [this](uint16_t http_status, uint8_t rcode, bool parse_ok, uint64_t cert_expiry_epoch, visor::http::HttpTimings t, const std::string &name, timespec stamp) { _doh_result_cb(http_status, rcode, parse_ok, cert_expiry_epoch, t, name, stamp); }); @@ -627,16 +749,31 @@ void NetProbeInputStream::stop() void scrub_netprobe_config_json(json &cfg) { // Scrub every config value that can carry a secret from a raw config echo: proxy URLs can - // embed credentials, and header/body/expected_body(_regex) values can be anything the operator - // configured (Authorization headers, tokens in a probe body, etc.). Values must NEVER appear - // in any serialized config — only header/target NAMES are safe. Used by BOTH the input + // embed credentials, and header/body/expected_body(_regex)/not_contains/body_not_matches_regex/ + // json_equals/fail_if_header_(not_)matches values can be anything the operator configured + // (Authorization headers, tokens in a probe body or assertion pattern, etc.). Values must NEVER + // appear in any serialized config — only header/target NAMES are safe. Used by BOTH the input // stream's info_json (module config echo) and the netprobe input plugin's redact hook (tap // config echo via Tap::info_json — GET /api/v1/taps and Policy::info_json). - for (const char *key : {"proxy", "body", "expected_body", "expected_body_regex"}) { + for (const char *key : {"proxy", "body", "expected_body", "expected_body_regex", + "not_contains", "body_not_matches_regex", "json_equals"}) { if (cfg.contains(key)) { cfg[key] = ""; } } + for (const char *hkey : {"fail_if_header_matches", "fail_if_header_not_matches"}) { + if (cfg.contains(hkey)) { + if (cfg[hkey].is_object()) { + for (auto &el : cfg[hkey].items()) { + el.value() = ""; + } + } else { + // Malformed shape (scalar/list rather than the expected map) is still potentially + // secret-bearing — redact the whole key rather than leave a raw value in the echo. + cfg[hkey] = ""; + } + } + } if (cfg.contains("targets") && cfg["targets"].is_object()) { for (auto &el : cfg["targets"].items()) { auto &tgt_val = el.value(); diff --git a/src/inputs/netprobe/NetProbeInputStream.h b/src/inputs/netprobe/NetProbeInputStream.h index 80929c038..d9682e1d5 100644 --- a/src/inputs/netprobe/NetProbeInputStream.h +++ b/src/inputs/netprobe/NetProbeInputStream.h @@ -49,6 +49,10 @@ class NetProbeInputStream : public visor::InputStream // (used to build the actual request) separate from just the NAMES (safe to echo in info_json). std::map> _http_target_headers; std::map> _http_target_header_names; + // per-target ip_version/resolve overrides (CURLOPT_IPRESOLVE/CURLOPT_RESOLVE), keyed like + // _http_targets/_doh_targets — used for both http and doh targets. + std::map _http_target_ipresolve; + std::map> _http_target_resolve; HttpProbeOptions _http_opts; std::map _doh_targets; std::string _doh_qname; @@ -89,7 +93,17 @@ class NetProbeInputStream : public visor::InputStream "body", "body_check_max_bytes", "proxy", - "tls"}; + "tls", + "json_path", + "json_equals", + "not_contains", + "body_not_matches_regex", + "min_response_size_bytes", + "max_response_size_bytes", + "fail_if_header_matches", + "fail_if_header_not_matches", + "max_last_modified_diff_secs", + "valid_http_versions"}; void _create_netprobe_loop(); void _send_cb(pcpp::Packet &, TestType, const std::string &, timespec); diff --git a/src/inputs/netprobe/test_netprobe.cpp b/src/inputs/netprobe/test_netprobe.cpp index 184d8ffc7..513b7c256 100644 --- a/src/inputs/netprobe/test_netprobe.cpp +++ b/src/inputs/netprobe/test_netprobe.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -128,7 +129,7 @@ TEST_CASE("Netprobe invalid config", "[netprobe][config]") NetProbeInputStream stream{"net-probe-test"}; stream.config_set("invalid_config", true); - CHECK_THROWS_WITH(stream.start(), "invalid_config is an invalid/unsupported config or filter. The valid configs/filters are: test_type, interval_msec, timeout_msec, packets_per_test, packets_interval_msec, packet_payload_size, targets, http_method, qname, qtype, expected_status, failure_status, expected_body, expected_body_regex, body, body_check_max_bytes, proxy, tls"); + CHECK_THROWS_WITH(stream.start(), "invalid_config is an invalid/unsupported config or filter. The valid configs/filters are: test_type, interval_msec, timeout_msec, packets_per_test, packets_interval_msec, packet_payload_size, targets, http_method, qname, qtype, expected_status, failure_status, expected_body, expected_body_regex, body, body_check_max_bytes, proxy, tls, json_path, json_equals, not_contains, body_not_matches_regex, min_response_size_bytes, max_response_size_bytes, fail_if_header_matches, fail_if_header_not_matches, max_last_modified_diff_secs, valid_http_versions"); } TEST_CASE("NetProbe ip_version config", "[netprobe][config][ipv6]") @@ -163,7 +164,7 @@ TEST_CASE("NetProbe ip_version config", "[netprobe][config][ipv6]") SECTION("top-level valid-keys string unchanged") { NetProbeInputStream s{"net-probe-test"}; s.config_set("invalid_config", true); - CHECK_THROWS_WITH(s.start(), "invalid_config is an invalid/unsupported config or filter. The valid configs/filters are: test_type, interval_msec, timeout_msec, packets_per_test, packets_interval_msec, packet_payload_size, targets, http_method, qname, qtype, expected_status, failure_status, expected_body, expected_body_regex, body, body_check_max_bytes, proxy, tls"); + CHECK_THROWS_WITH(s.start(), "invalid_config is an invalid/unsupported config or filter. The valid configs/filters are: test_type, interval_msec, timeout_msec, packets_per_test, packets_interval_msec, packet_payload_size, targets, http_method, qname, qtype, expected_status, failure_status, expected_body, expected_body_regex, body, body_check_max_bytes, proxy, tls, json_path, json_equals, not_contains, body_not_matches_regex, min_response_size_bytes, max_response_size_bytes, fail_if_header_matches, fail_if_header_not_matches, max_last_modified_diff_secs, valid_http_versions"); } } @@ -451,6 +452,244 @@ TEST_CASE("NetProbe v2 config: proxy and tls are not supported for tcp", "[netpr } } +// --------------------------------------------------------------------------- +// v3: stream-level response-assertion config — parse + validate. +// --------------------------------------------------------------------------- + +namespace { +// Shared helper: an http-typed stream with a single valid target, ready for v3 config keys to be +// layered on top before start() is called. +std::unique_ptr make_v3_http_stream(const std::string &name) +{ + auto s = std::make_unique(name); + s->config_set("test_type", "http"); + auto targets = std::make_shared(); + auto target = std::make_shared(); + target->config_set("target", std::string("https://example.com/")); + targets->config_set>("t", target); + s->config_set>("targets", targets); + return s; +} +} + +TEST_CASE("NetProbe v3 config: json_path must be a valid JSON Pointer", "[netprobe][config][http]") +{ + auto s = make_v3_http_stream("v3-json-path-bad"); + s->config_set("json_path", std::string("data/status")); // missing leading '/' + CHECK_THROWS_WITH(s->start(), Catch::Matchers::ContainsSubstring("json_path is not a valid JSON Pointer")); +} + +TEST_CASE("NetProbe v3 config: json_equals requires json_path", "[netprobe][config][http]") +{ + auto s = make_v3_http_stream("v3-json-equals-no-path"); + s->config_set("json_equals", std::string("ok")); + CHECK_THROWS_WITH(s->start(), "netprobe: 'json_equals' requires 'json_path'"); +} + +TEST_CASE("NetProbe v3 config: unclosed body_not_matches_regex is rejected without quoting the pattern", "[netprobe][config][http]") +{ + auto s = make_v3_http_stream("v3-body-not-matches-bad"); + s->config_set("body_not_matches_regex", std::string("(unclosed")); + CHECK_THROWS_WITH(s->start(), + Catch::Matchers::ContainsSubstring("body_not_matches_regex") && !Catch::Matchers::ContainsSubstring("(unclosed")); +} + +TEST_CASE("NetProbe v3 config: fail_if_header_matches with an invalid regex names the header, not the pattern", "[netprobe][config][http]") +{ + auto s = make_v3_http_stream("v3-header-matcher-bad"); + auto matchers = std::make_shared(); + matchers->config_set("X-Debug", std::string("(bad")); + s->config_set>("fail_if_header_matches", matchers); + CHECK_THROWS_WITH(s->start(), + Catch::Matchers::ContainsSubstring("value_regex") && Catch::Matchers::ContainsSubstring("X-Debug") && !Catch::Matchers::ContainsSubstring("(bad")); +} + +TEST_CASE("NetProbe v3 config: min_response_size_bytes must not exceed max_response_size_bytes", "[netprobe][config][http]") +{ + auto s = make_v3_http_stream("v3-size-bounds-bad"); + s->config_set("min_response_size_bytes", 100); + s->config_set("max_response_size_bytes", 10); + CHECK_THROWS_WITH(s->start(), "netprobe: min_response_size_bytes must not exceed max_response_size_bytes"); +} + +TEST_CASE("NetProbe v3 config: valid_http_versions rejects an unsupported entry", "[netprobe][config][http]") +{ + auto s = make_v3_http_stream("v3-http-version-bad"); + s->config_set("valid_http_versions", {"9"}); + CHECK_THROWS_WITH(s->start(), "netprobe: invalid valid_http_versions entry '9' (use 1.0, 1.1, 2, or 3)"); +} + +TEST_CASE("NetProbe v3 config: json_path is not supported for test_type 'doh'", "[netprobe][config][doh]") +{ + auto s = std::make_unique("v3-doh-json-path"); + s->config_set("test_type", "doh"); + s->config_set("qname", std::string("example.com")); + s->config_set("json_path", std::string("/status")); + auto targets = std::make_shared(); + auto target = std::make_shared(); + target->config_set("target", std::string("https://1.1.1.1/dns-query")); + targets->config_set>("t", target); + s->config_set>("targets", targets); + CHECK_THROWS_WITH(s->start(), "'json_path' is not supported for test_type 'doh'"); +} + +TEST_CASE("NetProbe v3 config: json_path/not_contains/size/version all parse successfully together", "[netprobe][config][http]") +{ + // Positive case: every v3 key is accepted and parses/validates without throwing. start() + // only builds the io loop and schedules probe timers — it does not perform any network I/O + // synchronously — so calling stop() immediately after, with no sleep, exercises the + // config-parse path only and never actually contacts the (non-existent) https://example.com/ + // target. + auto s = make_v3_http_stream("v3-all-keys-ok"); + s->config_set("json_path", std::string("/status")); + s->config_set("json_equals", std::string("ok")); + s->config_set("not_contains", std::string("error")); + s->config_set("body_not_matches_regex", std::string("fail-[0-9]+")); + s->config_set("min_response_size_bytes", 10); + s->config_set("max_response_size_bytes", 1000); + s->config_set("max_last_modified_diff_secs", 3600); + s->config_set("valid_http_versions", {"1.1", "2"}); + auto fm = std::make_shared(); + fm->config_set("X-Error", std::string("^true$")); + s->config_set>("fail_if_header_matches", fm); + auto fnm = std::make_shared(); + fnm->config_set("X-Ok", std::string("^true$")); + s->config_set>("fail_if_header_not_matches", fnm); + + CHECK_NOTHROW(s->start()); + s->stop(); +} + +TEST_CASE("NetProbe v3 scrub helper: redacts not_contains/json_equals/body_not_matches_regex/header-matcher secrets", "[netprobe][http][config]") +{ + // Mirrors the v2 scrub-helper test: feed a tap-shaped config JSON and prove every v3 + // secret-bearing value is masked (never quoted/echoed) while non-secret keys survive. + json cfg; + cfg["not_contains"] = "v3-not-contains-sekrit"; + cfg["json_equals"] = "v3-json-equals-sekrit"; + cfg["body_not_matches_regex"] = "v3-regex-sekrit"; + cfg["json_path"] = "/status"; // not a secret; survives untouched + cfg["fail_if_header_matches"]["X-Debug"] = "v3-header-matcher-sekrit"; + cfg["fail_if_header_not_matches"]["X-Ok"] = "v3-header-not-matcher-sekrit"; + + visor::input::netprobe::scrub_netprobe_config_json(cfg); + + auto dumped = cfg.dump(); + CHECK(dumped.find("v3-not-contains-sekrit") == std::string::npos); + CHECK(dumped.find("v3-json-equals-sekrit") == std::string::npos); + CHECK(dumped.find("v3-regex-sekrit") == std::string::npos); + CHECK(dumped.find("v3-header-matcher-sekrit") == std::string::npos); + CHECK(dumped.find("v3-header-not-matcher-sekrit") == std::string::npos); + // Names and non-secret values survive. + CHECK(dumped.find("X-Debug") != std::string::npos); + CHECK(dumped.find("X-Ok") != std::string::npos); + CHECK(cfg["json_path"] == "/status"); + CHECK(cfg["not_contains"] == ""); + CHECK(cfg["json_equals"] == ""); + CHECK(cfg["body_not_matches_regex"] == ""); + CHECK(cfg["fail_if_header_matches"]["X-Debug"] == ""); + CHECK(cfg["fail_if_header_not_matches"]["X-Ok"] == ""); +} + +TEST_CASE("NetProbe v3 scrub helper: a malformed (non-object) header-matcher value is fully redacted", "[netprobe][http][config]") +{ + // The header matchers are normally maps (header-name: value_regex). If a config arrives with + // the key set to a scalar or list instead, the value can still carry a secret — redact the + // whole key rather than echo the raw value. + json cfg; + cfg["fail_if_header_matches"] = "v3-scalar-matcher-sekrit"; // malformed: scalar + cfg["fail_if_header_not_matches"] = json::array({"v3-list-matcher-sekrit"}); // malformed: list + + visor::input::netprobe::scrub_netprobe_config_json(cfg); + + auto dumped = cfg.dump(); + CHECK(dumped.find("v3-scalar-matcher-sekrit") == std::string::npos); + CHECK(dumped.find("v3-list-matcher-sekrit") == std::string::npos); + CHECK(cfg["fail_if_header_matches"] == ""); + CHECK(cfg["fail_if_header_not_matches"] == ""); +} + +// --------------------------------------------------------------------------- +// v3: per-target ip_version/resolve — parse + validate (threaded into probe ctors, +// evaluated by libcurl itself; not asserted here). +// --------------------------------------------------------------------------- + +TEST_CASE("NetProbe v3 config: per-target resolve entry must be host:port:address", "[netprobe][config][http]") +{ + NetProbeInputStream stream{"net-probe-test-resolve-bad"}; + stream.config_set("test_type", "http"); + auto targets = std::make_shared(); + auto target = std::make_shared(); + target->config_set("target", std::string("https://example.com/")); + target->config_set("resolve", {"bad-no-colons"}); + targets->config_set>("t", target); + stream.config_set>("targets", targets); + CHECK_THROWS_WITH(stream.start(), "netprobe: target 't' has an invalid resolve entry 'bad-no-colons' (expected host:port:address)"); +} + +TEST_CASE("NetProbe v3 config: per-target ip_version must be 4 or 6", "[netprobe][config][http]") +{ + NetProbeInputStream stream{"net-probe-test-ipver-bad"}; + stream.config_set("test_type", "http"); + auto targets = std::make_shared(); + auto target = std::make_shared(); + target->config_set("target", std::string("https://example.com/")); + target->config_set("ip_version", 5); + targets->config_set>("t", target); + stream.config_set>("targets", targets); + CHECK_THROWS_WITH(stream.start(), "ip_version must be 4 or 6"); +} + +TEST_CASE("NetProbe v3 config: per-target ip_version + resolve happy path reaches start", "[netprobe][config][http]") +{ + NetProbeInputStream stream{"net-probe-test-ipver-resolve-ok"}; + stream.config_set("test_type", "http"); + auto targets = std::make_shared(); + auto target = std::make_shared(); + target->config_set("target", std::string("https://example.com/")); + target->config_set("ip_version", 4); + target->config_set("resolve", {"example.com:443:127.0.0.1"}); + targets->config_set>("t", target); + stream.config_set>("targets", targets); + + CHECK_NOTHROW(stream.start()); + stream.stop(); +} + +TEST_CASE("NetProbe v3 config: per-target resolve accepts an (unbracketed) IPv6 address", "[netprobe][config][http]") +{ + // The address field may itself contain colons (IPv6). Validation keys off the first two colons + // (host, port) and must accept the IPv6 remainder; bracketing for curl happens in the transport. + NetProbeInputStream stream{"net-probe-test-resolve-v6"}; + stream.config_set("test_type", "http"); + auto targets = std::make_shared(); + auto target = std::make_shared(); + target->config_set("target", std::string("https://example.com/")); + target->config_set("resolve", {"example.com:443:2001:db8::1"}); + targets->config_set>("t", target); + stream.config_set>("targets", targets); + + CHECK_NOTHROW(stream.start()); + stream.stop(); +} + +TEST_CASE("NetProbe v3 config: per-target ip_version + resolve also parse for doh targets", "[netprobe][config][doh]") +{ + NetProbeInputStream stream{"net-probe-test-doh-ipver-resolve-ok"}; + stream.config_set("test_type", "doh"); + stream.config_set("qname", std::string("example.com")); + auto targets = std::make_shared(); + auto target = std::make_shared(); + target->config_set("target", std::string("https://1.1.1.1/dns-query")); + target->config_set("ip_version", 6); + target->config_set("resolve", {"1.1.1.1:443:127.0.0.1"}); + targets->config_set>("t", target); + stream.config_set>("targets", targets); + + CHECK_NOTHROW(stream.start()); + stream.stop(); +} + TEST_CASE("ICMPv6 reply carrier survives the fan-out Packet deep-copy", "[netprobe][ipv6]") { // Wire bytes of an ICMPv6 echo REPLY: type=129, code=0, checksum=0, id=0xBEEF, seq=0x0102 (network order). @@ -1437,6 +1676,493 @@ TEST_CASE("NetProbe HTTP e2e v2: body match past the capture cap is not a false CHECK(tgt["content_failures"].get() == 0); } +TEST_CASE("NetProbe HTTP e2e v3: truncated body still lets a size assertion fail", "[netprobe][http][e2e]") +{ + // The AND-fold's subtle case: a truncated body SKIPS body assertions, but a NON-body assertion + // (size, which uses the true downloaded size) must still run and can fail. Body is ~64 KB but + // body_check_max_bytes caps capture at 1 KB (so the body assertion is skipped); max_response_size + // is far below the true size, so the size check fails => content_failures, not a false success. + httplib::Server svr; + std::string body(64 * 1024, 'x'); + svr.Get("/health", [&](const httplib::Request &, httplib::Response &res) { + res.set_content(body, "text/plain"); + }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread server_thread([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, server_thread}; + svr.wait_until_ready(); + + std::string url = "http://127.0.0.1:" + std::to_string(port) + "/health"; + NetProbeInputStream stream{"netprobe-http-e2e-trunc-size"}; + stream.config_set("test_type", "http"); + stream.config_set("interval_msec", 200); + stream.config_set("timeout_msec", 150); + stream.config_set("expected_body", std::string("never-present")); // body assertion — will be SKIPPED (truncated) + stream.config_set("body_check_max_bytes", 1024); // truncate the captured body to 1 KB + // max sits BETWEEN the capture cap (1024) and the true size (64 KB): a captured-length comparison + // would PASS (1024 <= 2048) but the true size (65536) FAILS — so this pins that the size check + // uses r.response_size (true downloaded size), not the truncated captured length. + stream.config_set("max_response_size_bytes", 2048); + auto targets = std::make_shared(); + auto target = std::make_shared(); + target->config_set("target", url); + targets->config_set>("health_target", target); + stream.config_set>("targets", targets); + + visor::Config c; + c.config_set("num_periods", 1); + auto *proxy = stream.add_event_proxy(c); + NetProbeStreamHandler handler{"netprobe-http-e2e-trunc-size", proxy, &c}; + + handler.start(); + stream.start(); + std::this_thread::sleep_for(750ms); + stream.stop(); + handler.stop(); + + json j; + handler.metrics()->bucket(0)->to_json(j); + REQUIRE(j["targets"].contains("health_target")); + auto &tgt = j["targets"]["health_target"]; + CHECK(tgt["attempts"].get() >= 1); + // Size assertion ran despite the truncated body and failed => content_failures, NOT a false success. + CHECK(tgt["successes"].get() == 0); + CHECK(tgt["content_failures"].get() >= 1); +} + +// Helper: run one http probe stream against `url` with `configure(stream)` applied, for ~750ms, +// and return the target's metrics JSON node (target key "t"). +static json run_v3_http_probe(const std::string &name, const std::string &url, + const std::function &configure) +{ + NetProbeInputStream stream{name}; + stream.config_set("test_type", "http"); + stream.config_set("interval_msec", 200); + stream.config_set("timeout_msec", 150); + configure(stream); + auto targets = std::make_shared(); + auto target = std::make_shared(); + target->config_set("target", url); + targets->config_set>("t", target); + stream.config_set>("targets", targets); + + visor::Config c; + c.config_set("num_periods", 1); + auto *proxy = stream.add_event_proxy(c); + NetProbeStreamHandler handler{name, proxy, &c}; + handler.start(); + stream.start(); + std::this_thread::sleep_for(750ms); + stream.stop(); + handler.stop(); + json j; + handler.metrics()->bucket(0)->to_json(j); + return j["targets"].contains("t") ? j["targets"]["t"] : json::object(); +} + +TEST_CASE("NetProbe HTTP e2e v3: json_path pass and fail", "[netprobe][http][e2e]") +{ + httplib::Server svr; + svr.Get("/j", [](const httplib::Request &, httplib::Response &res) { + res.set_content(R"({"data":{"status":"ok"}})", "application/json"); + }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread th([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, th}; + svr.wait_until_ready(); + std::string url = "http://127.0.0.1:" + std::to_string(port) + "/j"; + + auto pass = run_v3_http_probe("v3-json-pass", url, [](NetProbeInputStream &s) { + s.config_set("json_path", std::string("/data/status")); + s.config_set("json_equals", std::string("ok")); + }); + CHECK(pass["successes"].get() >= 1); + CHECK(pass["content_failures"].get() == 0); + + auto fail = run_v3_http_probe("v3-json-fail", url, [](NetProbeInputStream &s) { + s.config_set("json_path", std::string("/data/status")); + s.config_set("json_equals", std::string("down")); + }); + CHECK(fail["successes"].get() == 0); + CHECK(fail["content_failures"].get() >= 1); +} + +TEST_CASE("NetProbe HTTP e2e v3: not_contains fail", "[netprobe][http][e2e]") +{ + httplib::Server svr; + svr.Get("/b", [](const httplib::Request &, httplib::Response &res) { + res.set_content("python traceback (most recent call last)", "text/plain"); + }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread th([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, th}; + svr.wait_until_ready(); + auto tgt = run_v3_http_probe("v3-not-contains", "http://127.0.0.1:" + std::to_string(port) + "/b", + [](NetProbeInputStream &s) { s.config_set("not_contains", std::string("traceback")); }); + CHECK(tgt["successes"].get() == 0); + CHECK(tgt["content_failures"].get() >= 1); +} + +TEST_CASE("NetProbe HTTP e2e v3: size bound fail", "[netprobe][http][e2e]") +{ + httplib::Server svr; + svr.Get("/s", [](const httplib::Request &, httplib::Response &res) { + res.set_content("tiny", "text/plain"); + }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread th([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, th}; + svr.wait_until_ready(); + auto tgt = run_v3_http_probe("v3-size", "http://127.0.0.1:" + std::to_string(port) + "/s", + [](NetProbeInputStream &s) { s.config_set("min_response_size_bytes", 100000); }); + CHECK(tgt["successes"].get() == 0); + CHECK(tgt["content_failures"].get() >= 1); +} + +TEST_CASE("NetProbe HTTP e2e v3: header matchers", "[netprobe][http][e2e]") +{ + httplib::Server svr; + svr.Get("/h", [](const httplib::Request &, httplib::Response &res) { + res.set_header("X-Debug", "1"); + res.set_content("{}", "application/json"); + }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread th([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, th}; + svr.wait_until_ready(); + std::string url = "http://127.0.0.1:" + std::to_string(port) + "/h"; + + // fail_if_header_matches: X-Debug present with any value => content failure + auto fm = run_v3_http_probe("v3-hdr-fail", url, [](NetProbeInputStream &s) { + auto m = std::make_shared(); + m->config_set("X-Debug", std::string(".+")); + s.config_set>("fail_if_header_matches", m); + }); + CHECK(fm["successes"].get() == 0); + CHECK(fm["content_failures"].get() >= 1); + + // fail_if_header_not_matches: Content-Type must match application/json (it does) => success + auto fnm = run_v3_http_probe("v3-hdr-pass", url, [](NetProbeInputStream &s) { + auto m = std::make_shared(); + m->config_set("Content-Type", std::string("application/json")); + s.config_set>("fail_if_header_not_matches", m); + }); + CHECK(fnm["successes"].get() >= 1); + CHECK(fnm["content_failures"].get() == 0); +} + +TEST_CASE("NetProbe HTTP e2e v3: header matcher fails conservatively when header capture truncates", "[netprobe][http][e2e]") +{ + // The response carries so many headers that capture stops at the byte cap (headers_truncated). + // fail_if_header_matches targets a header that is NOT among the captured ones, so matches() + // alone would PASS — but because a forbidden header could be hiding in the dropped tail, the + // probe must fail the assertion conservatively. + httplib::Server svr; + svr.Get("/many", [](const httplib::Request &, httplib::Response &res) { + for (int i = 0; i < 1400; ++i) { + res.set_header("X-Pad-" + std::to_string(i), std::string(48, 'y')); + } + res.set_content("ok", "text/plain"); + }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread th([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, th}; + svr.wait_until_ready(); + std::string url = "http://127.0.0.1:" + std::to_string(port) + "/many"; + + auto tgt = run_v3_http_probe("v3-hdr-trunc", url, [](NetProbeInputStream &s) { + auto m = std::make_shared(); + m->config_set("X-Absent", std::string(".+")); // never present => matches() would pass + s.config_set>("fail_if_header_matches", m); + }); + CHECK(tgt["successes"].get() == 0); + CHECK(tgt["content_failures"].get() >= 1); +} + +TEST_CASE("NetProbe HTTP e2e v3: required-header check passes despite header truncation", "[netprobe][http][e2e]") +{ + // A fail_if_header_not_matches (required-header) PASS is a positive presence proof: the required + // header was FOUND in the captured headers. Dropping headers past the 64 KiB cap cannot un-find + // it, so truncation must NOT fail this check. + // + // Robust, order-independent construction (httplib stores headers in an unordered_multimap, so + // iteration order is NOT alphabetical): three ~40 KiB pad headers guarantee truncation, and + // because any TWO of them already exceed the 64 KiB cap, at most ONE pad is ever admitted before + // capture stops growing — so the tiny "A-Required" line always fits with a ~24 KiB margin no + // matter where it lands in the header order. + httplib::Server svr; + svr.Get("/many", [](const httplib::Request &, httplib::Response &res) { + res.set_header("A-Required", "yes"); + for (int i = 0; i < 3; ++i) { + res.set_header("X-Pad-" + std::to_string(i), std::string(40000, 'y')); + } + res.set_content("ok", "text/plain"); + }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread th([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, th}; + svr.wait_until_ready(); + std::string url = "http://127.0.0.1:" + std::to_string(port) + "/many"; + + auto tgt = run_v3_http_probe("v3-hdr-req-trunc", url, [](NetProbeInputStream &s) { + auto m = std::make_shared(); + m->config_set("A-Required", std::string("yes")); // present & matched pre-cap => real PASS + s.config_set>("fail_if_header_not_matches", m); + }); + CHECK(tgt["successes"].get() >= 1); + CHECK(tgt["content_failures"].get() == 0); +} + +TEST_CASE("NetProbe HTTP e2e v3: truncation still fails when forbidden rules coexist with required", "[netprobe][http][e2e]") +{ + // When BOTH a required (fail_if_header_not_matches, satisfied) and a forbidden + // (fail_if_header_matches, not seen) rule are configured, truncation must still fail: a forbidden + // header could be hiding in the dropped tail. has_forbidden_rules() keeps the fail-safe on here. + // Same order-independent construction as the required-only test (at most one ~40 KiB pad is + // admitted, so "A-Required" is always captured while truncation is still triggered). + httplib::Server svr; + svr.Get("/many", [](const httplib::Request &, httplib::Response &res) { + res.set_header("A-Required", "yes"); + for (int i = 0; i < 3; ++i) { + res.set_header("X-Pad-" + std::to_string(i), std::string(40000, 'y')); + } + res.set_content("ok", "text/plain"); + }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread th([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, th}; + svr.wait_until_ready(); + std::string url = "http://127.0.0.1:" + std::to_string(port) + "/many"; + + auto tgt = run_v3_http_probe("v3-hdr-both-trunc", url, [](NetProbeInputStream &s) { + auto req = std::make_shared(); + req->config_set("A-Required", std::string("yes")); + s.config_set>("fail_if_header_not_matches", req); + auto forbid = std::make_shared(); + forbid->config_set("X-Absent", std::string(".+")); // absent from captured set => matches() PASS + s.config_set>("fail_if_header_matches", forbid); + }); + CHECK(tgt["successes"].get() == 0); + CHECK(tgt["content_failures"].get() >= 1); +} + +TEST_CASE("NetProbe HTTP e2e v3: valid_http_versions rejects the negotiated version", "[netprobe][http][e2e]") +{ + httplib::Server svr; // plain httplib serves HTTP/1.1 + svr.Get("/v", [](const httplib::Request &, httplib::Response &res) { + res.set_content("ok", "text/plain"); + }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread th([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, th}; + svr.wait_until_ready(); + auto tgt = run_v3_http_probe("v3-httpver", "http://127.0.0.1:" + std::to_string(port) + "/v", + [](NetProbeInputStream &s) { s.config_set("valid_http_versions", {"2"}); }); + // Negotiated 1.1 is not in {"2"} => content failure. + CHECK(tgt["successes"].get() == 0); + CHECK(tgt["content_failures"].get() >= 1); +} + +TEST_CASE("NetProbe HTTP e2e v3: header assertion sees only the FINAL response across a redirect", "[netprobe][http][e2e]") +{ + // Server A (301) carries X-Debug that WOULD trip fail_if_header_matches; server B (200) does not. + // No request body/custom request headers => redirects are followed. The per-response capture + // reset must keep only B's headers, so content_failures stays 0. + httplib::Server svr_b; + svr_b.Get("/final", [](const httplib::Request &, httplib::Response &res) { + res.set_content("ok", "text/plain"); // no X-Debug on the final response + }); + int port_b = svr_b.bind_to_any_port("127.0.0.1"); + REQUIRE(port_b > 0); + std::thread tb([&svr_b] { svr_b.listen_after_bind(); }); + ServerGuard gb{svr_b, tb}; + svr_b.wait_until_ready(); + + httplib::Server svr_a; + std::string loc = "http://127.0.0.1:" + std::to_string(port_b) + "/final"; + svr_a.Get("/redirect", [&](const httplib::Request &, httplib::Response &res) { + res.status = 301; + res.set_header("X-Debug", "intermediate"); // only on the intermediate response + res.set_header("Location", loc); + }); + int port_a = svr_a.bind_to_any_port("127.0.0.1"); + REQUIRE(port_a > 0); + std::thread ta([&svr_a] { svr_a.listen_after_bind(); }); + ServerGuard ga{svr_a, ta}; + svr_a.wait_until_ready(); + + auto tgt = run_v3_http_probe("v3-redirect-hdr", "http://127.0.0.1:" + std::to_string(port_a) + "/redirect", + [](NetProbeInputStream &s) { + auto m = std::make_shared(); + m->config_set("X-Debug", std::string(".+")); + s.config_set>("fail_if_header_matches", m); + }); + // X-Debug lived only on the 301; the final 200 is clean => no content failure. + CHECK(tgt["successes"].get() >= 1); + CHECK(tgt["content_failures"].get() == 0); +} + +TEST_CASE("NetProbe HTTP e2e v3: not_contains hit in a truncated prefix still fails", "[netprobe][http][e2e]") +{ + // The forbidden token sits in the FIRST bytes of a large body. Even though the body is truncated + // at the capture cap, a visible not_contains hit is a definitive failure (its presence does not + // depend on the dropped tail) — must be content_failures, never a false success. + httplib::Server svr; + std::string body = std::string("traceback: boom\n") + std::string(64 * 1024, 'x'); + svr.Get("/e", [&](const httplib::Request &, httplib::Response &res) { + res.set_content(body, "text/plain"); + }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread th([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, th}; + svr.wait_until_ready(); + auto tgt = run_v3_http_probe("v3-trunc-notcontains", "http://127.0.0.1:" + std::to_string(port) + "/e", + [](NetProbeInputStream &s) { + s.config_set("not_contains", std::string("traceback")); + s.config_set("body_check_max_bytes", 1024); // truncates, but 'traceback' is in the prefix + }); + CHECK(tgt["successes"].get() == 0); + CHECK(tgt["content_failures"].get() >= 1); +} + +TEST_CASE("NetProbe HTTP e2e v3: max_response_size_bytes 0 requires an empty body", "[netprobe][http][e2e]") +{ + // 0 is a real bound (require empty body), not "unset": a non-empty response must fail, an empty + // response must pass. + httplib::Server svr; + svr.Get("/nonempty", [](const httplib::Request &, httplib::Response &res) { res.set_content("x", "text/plain"); }); + svr.Get("/empty", [](const httplib::Request &, httplib::Response &res) { res.set_content("", "text/plain"); }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread th([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, th}; + svr.wait_until_ready(); + std::string base = "http://127.0.0.1:" + std::to_string(port); + + auto ne = run_v3_http_probe("v3-maxsize0-ne", base + "/nonempty", + [](NetProbeInputStream &s) { s.config_set("max_response_size_bytes", 0); }); + CHECK(ne["successes"].get() == 0); + CHECK(ne["content_failures"].get() >= 1); + + auto em = run_v3_http_probe("v3-maxsize0-e", base + "/empty", + [](NetProbeInputStream &s) { s.config_set("max_response_size_bytes", 0); }); + CHECK(em["successes"].get() >= 1); + CHECK(em["content_failures"].get() == 0); +} + +TEST_CASE("NetProbe HTTP e2e v3: stale Last-Modified fails max_last_modified_diff_secs", "[netprobe][http][e2e]") +{ + httplib::Server svr; + svr.Get("/lm", [](const httplib::Request &, httplib::Response &res) { + res.set_header("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"); // ancient => older than the diff + res.set_content("ok", "text/plain"); + }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread th([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, th}; + svr.wait_until_ready(); + auto tgt = run_v3_http_probe("v3-lastmod", "http://127.0.0.1:" + std::to_string(port) + "/lm", + [](NetProbeInputStream &s) { s.config_set("max_last_modified_diff_secs", 3600); }); + CHECK(tgt["successes"].get() == 0); + CHECK(tgt["content_failures"].get() >= 1); +} + +TEST_CASE("NetProbe HTTP e2e v3: per-target resolve pins a bogus host to the test server", "[netprobe][http][e2e]") +{ + // Proves CURLOPT_RESOLVE is applied: the target host "bogus.invalid" would never resolve via DNS, + // so a recorded success can only happen if the resolve override mapped it to 127.0.0.1. + httplib::Server svr; + svr.Get("/ok", [](const httplib::Request &, httplib::Response &res) { res.set_content("ok", "text/plain"); }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread th([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, th}; + svr.wait_until_ready(); + + NetProbeInputStream stream{"v3-resolve"}; + stream.config_set("test_type", "http"); + stream.config_set("interval_msec", 200); + stream.config_set("timeout_msec", 150); + auto targets = std::make_shared(); + auto target = std::make_shared(); + target->config_set("target", std::string("http://bogus.invalid:" + std::to_string(port) + "/ok")); + target->config_set("resolve", + {"bogus.invalid:" + std::to_string(port) + ":127.0.0.1"}); + targets->config_set>("t", target); + stream.config_set>("targets", targets); + + visor::Config c; + c.config_set("num_periods", 1); + auto *proxy = stream.add_event_proxy(c); + NetProbeStreamHandler handler{"v3-resolve", proxy, &c}; + handler.start(); + stream.start(); + std::this_thread::sleep_for(750ms); + stream.stop(); + handler.stop(); + json j; + handler.metrics()->bucket(0)->to_json(j); + REQUIRE(j["targets"].contains("t")); + CHECK(j["targets"]["t"]["successes"].get() >= 1); +} + +TEST_CASE("NetProbe HTTP e2e v3: a per-target resolve override does not leak to another target", "[netprobe][http][e2e]") +{ + // Two targets for the SAME host:port on one stream (shared curl_multi). "pinned" overrides + // bogus.invalid -> 127.0.0.1 (the test server); "unpinned" has NO override and must NOT be + // silently sent to the pinned address. With CONNECT_TO (per-handle, no shared DNS cache) the + // unpinned target does real DNS for bogus.invalid and fails — proving the override is scoped. + httplib::Server svr; + svr.Get("/ok", [](const httplib::Request &, httplib::Response &res) { res.set_content("ok", "text/plain"); }); + int port = svr.bind_to_any_port("127.0.0.1"); + REQUIRE(port > 0); + std::thread th([&svr] { svr.listen_after_bind(); }); + ServerGuard guard{svr, th}; + svr.wait_until_ready(); + std::string ps = std::to_string(port); + + NetProbeInputStream stream{"v3-resolve-leak"}; + stream.config_set("test_type", "http"); + stream.config_set("interval_msec", 200); + stream.config_set("timeout_msec", 150); + auto targets = std::make_shared(); + auto pinned = std::make_shared(); + pinned->config_set("target", std::string("http://bogus.invalid:" + ps + "/ok")); + pinned->config_set("resolve", {"bogus.invalid:" + ps + ":127.0.0.1"}); + targets->config_set>("pinned", pinned); + auto unpinned = std::make_shared(); + unpinned->config_set("target", std::string("http://bogus.invalid:" + ps + "/ok")); // same host:port, NO resolve + targets->config_set>("unpinned", unpinned); + stream.config_set>("targets", targets); + + visor::Config c; + c.config_set("num_periods", 1); + auto *proxy = stream.add_event_proxy(c); + NetProbeStreamHandler handler{"v3-resolve-leak", proxy, &c}; + handler.start(); + stream.start(); + std::this_thread::sleep_for(750ms); + stream.stop(); + handler.stop(); + json j; + handler.metrics()->bucket(0)->to_json(j); + REQUIRE(j["targets"].contains("pinned")); + REQUIRE(j["targets"].contains("unpinned")); + CHECK(j["targets"]["pinned"]["successes"].get() >= 1); + CHECK(j["targets"]["unpinned"]["successes"].get() == 0); // override must NOT have leaked +} + TEST_CASE("NetProbe HTTP e2e v2: custom headers are not forwarded across redirects", "[netprobe][http][e2e]") { // Server A redirects to server B; B records whether it ever received the secret header. With a