feat(netprobe): HTTP/DoH probes v2 — success criteria, TLS cert expiry, headers/proxy/mTLS

conanfile.py

@@ -27,8 +27,8 @@ def requirements(self):
         self.requires("uvw/3.4.0")
         self.requires("yaml-cpp/0.9.0")
         self.requires("robin-hood-hashing/3.11.5")
-        self.requires("libcurl/8.20.0")
-        self.requires("libnghttp2/1.61.0")
+        self.requires("libcurl/8.21.0")
+        self.requires("libnghttp2/1.68.1")
         if (
             "libc" not in self.settings.compiler.fields
             or self.settings.compiler.libc != "musl"

libs/visor_http_client/CMakeLists.txt

@@ -5,7 +5,7 @@ find_package(uvw REQUIRED)
 find_package(httplib REQUIRED)
 find_package(Catch2 REQUIRED)
 
-add_library(VisorHttpClient STATIC HttpClient.cpp)
+add_library(VisorHttpClient STATIC HttpClient.cpp HttpCheck.cpp)
 # Namespaced alias for consistency with the other libs (Visor::Lib::Dns, Visor::Lib::Tcp, ...)
 # and cleaner downstream consumption.
 add_library(Visor::Lib::Http ALIAS VisorHttpClient)
@@ -15,3 +15,7 @@ target_link_libraries(VisorHttpClient PUBLIC CURL::libcurl uvw::uvw)
 add_executable(unit-tests-visor-http-client test_http_client.cpp)
 target_link_libraries(unit-tests-visor-http-client PRIVATE VisorHttpClient Catch2::Catch2WithMain httplib::httplib)
 add_test(NAME unit-tests-visor-http-client COMMAND unit-tests-visor-http-client)
+
+add_executable(unit-tests-visor-http-check test_http_check.cpp)
+target_link_libraries(unit-tests-visor-http-check PRIVATE VisorHttpClient Catch2::Catch2WithMain)
+add_test(NAME unit-tests-visor-http-check COMMAND unit-tests-visor-http-check)

libs/visor_http_client/HttpCheck.cpp

@@ -0,0 +1,103 @@
+#include "HttpCheck.h"
+#include <curl/curl.h> // curl_getdate (cpp only — the header stays curl-free)
+#include <stdexcept>
+
+namespace visor::http {
+
+static void set_range(std::vector<bool> &codes, unsigned lo, unsigned hi, const std::string &entry)
+{
+    if (lo < 100 || hi > 599 || lo > hi) {
+        throw std::invalid_argument("invalid status entry '" + entry + "' (codes must be 100-599, ranges low-high)");
+    }
+    for (unsigned c = lo; c <= hi; ++c) {
+        codes[c] = true;
+    }
+}
+
+// Parse a decimal status code from `s`, requiring the whole string to be consumed and the value
+// to be a plausible HTTP status (<= 599). Validating the unsigned-long result BEFORE narrowing to
+// unsigned is essential: a value that fits in unsigned long but exceeds unsigned (e.g. 4294967496
+// on LP64) would otherwise wrap to a small in-range code and be accepted. `entry` names the
+// offending token in the error.
+static unsigned parse_status_code(const std::string &s, const std::string &entry)
+{
+    try {
+        size_t pos{};
+        unsigned long v = std::stoul(s, &pos);
+        if (pos != s.size() || v > 599) {
+            throw std::invalid_argument(entry);
+        }
+        return static_cast<unsigned>(v);
+    } catch (const std::exception &) {
+        throw std::invalid_argument("invalid status entry '" + entry + "'");
+    }
+}
+
+StatusMatcher StatusMatcher::parse(const std::vector<std::string> &entries)
+{
+    StatusMatcher m;
+    for (const auto &e : entries) {
+        if (e.size() == 3 && (e[1] == 'x' || e[1] == 'X') && (e[2] == 'x' || e[2] == 'X') && e[0] >= '1' && e[0] <= '5') {
+            unsigned cls = static_cast<unsigned>(e[0] - '0');
+            set_range(m._codes, cls * 100, cls * 100 + 99, e);
+        } else if (auto dash = e.find('-'); dash != std::string::npos && dash > 0 && dash < e.size() - 1) {
+            unsigned lo = parse_status_code(e.substr(0, dash), e);
+            unsigned hi = parse_status_code(e.substr(dash + 1), e);
+            set_range(m._codes, lo, hi, e);
+        } else {
+            unsigned code = parse_status_code(e, e);
+            set_range(m._codes, code, code, e);
+        }
+        m._empty = false;
+    }
+    return m;
+}
+
+bool StatusMatcher::matches(uint16_t status) const
+{
+    return !_empty && status < _codes.size() && _codes[status];
+}
+
+bool StatusMatcher::empty() const
+{
+    return _empty;
+}
+
+BodyCheck BodyCheck::compile(const std::string &substr, const std::string &regex_pattern)
+{
+    BodyCheck b;
+    b.substring = substr;
+    if (!regex_pattern.empty()) {
+        try {
+            b.regex.emplace(regex_pattern, std::regex::ECMAScript);
+        } catch (const std::regex_error &) {
+            // never quote the pattern — it can embed secrets
+            throw std::invalid_argument("expected_body_regex is not a valid ECMAScript regular expression");
+        }
+    }
+    return b;
+}
+
+bool BodyCheck::matches(const std::string &body) const
+{
+    if (!substring.empty() && body.find(substring) == std::string::npos) {
+        return false;
+    }
+    if (regex.has_value() && !std::regex_search(body, *regex)) {
+        return false;
+    }
+    return true;
+}
+
+uint64_t parse_cert_expire_date(const std::string &date_str)
+{
+    // curl CERTINFO format, e.g. "Aug 15 12:00:00 2026 GMT" (day may be space-padded).
+    // curl_getdate() is token-based (handles month-name/day/time/year/zone in any order) and,
+    // unlike strptime/timegm, is fully portable incl. MSVC — netprobe builds on win64.
+    if (date_str.empty()) {
+        return 0;
+    }
+    time_t t = curl_getdate(date_str.c_str(), nullptr);
+    return t > 0 ? static_cast<uint64_t>(t) : 0;
+}
+}

libs/visor_http_client/HttpCheck.h

@@ -0,0 +1,38 @@
+#pragma once
+#include <cstdint>
+#include <optional>
+#include <regex>
+#include <string>
+#include <vector>
+
+namespace visor::http {
+
+// Parsed set of HTTP status codes (entries: "NNN", "Nxx", "A-B"). Throws std::invalid_argument
+// naming the bad ENTRY (never other config values) on grammar violations.
+class StatusMatcher
+{
+public:
+    StatusMatcher() = default;                              // empty (matches nothing); empty() == true
+    static StatusMatcher parse(const std::vector<std::string> &entries); // throws std::invalid_argument
+    bool matches(uint16_t status) const;
+    bool empty() const;
+
+private:
+    std::vector<bool> _codes = std::vector<bool>(600, false); // index by status; 100..599 valid
+    bool _empty{true};
+};
+
+// Body-content checks: substring AND regex (each optional). compile() throws std::invalid_argument
+// on a bad regex (message must NOT quote the pattern — patterns can embed secrets).
+struct BodyCheck {
+    std::string substring;                 // empty => not checked
+    std::optional<std::regex> regex;       // nullopt => not checked
+    bool configured() const { return !substring.empty() || regex.has_value(); }
+    static BodyCheck compile(const std::string &substr, const std::string &regex_pattern); // "" => absent
+    bool matches(const std::string &body) const; // AND of the configured checks
+};
+
+// Parse a curl CERTINFO "Expire date:" value, e.g. "Aug 15 12:00:00 2026 GMT", to unix epoch.
+// Returns 0 on parse failure. (Pure string->epoch; the CERTINFO iteration lives in HttpClient.)
+uint64_t parse_cert_expire_date(const std::string &date_str);
+}

libs/visor_http_client/HttpClient.cpp

@@ -1,4 +1,6 @@
 #include "HttpClient.h"
+#include "HttpCheck.h"
+#include <cstring>
 #include <mutex>
 #include <stdexcept>
 #include <utility>
@@ -23,6 +25,36 @@ static void ensure_curl_global_init()
     std::call_once(flag, [] { curl_global_init(CURL_GLOBAL_DEFAULT); });
 }
 
+// Replace every occurrence of `secret` in `s` with a placeholder (no-op if secret is empty).
+static void redact_secret(std::string &s, const std::string &secret)
+{
+    if (secret.empty()) {
+        return;
+    }
+    static const std::string rep = "<redacted>";
+    for (size_t pos = s.find(secret); pos != std::string::npos; pos = s.find(secret, pos + rep.size())) {
+        s.replace(pos, secret.size(), rep);
+    }
+}
+
+// Extract the "user:pass" userinfo from a proxy URL string, or "" if none. Userinfo is the span
+// between an optional "scheme://" and the "@" that terminates the authority's userinfo.
+static std::string proxy_userinfo(const std::string &proxy)
+{
+    size_t start = 0;
+    if (auto scheme = proxy.find("://"); scheme != std::string::npos) {
+        start = scheme + 3;
+    }
+    auto at = proxy.find('@', start);
+    if (at == std::string::npos) {
+        return "";
+    }
+    if (auto slash = proxy.find('/', start); slash != std::string::npos && slash < at) {
+        return ""; // '@' is past the authority (e.g. in a path) — not userinfo
+    }
+    return proxy.substr(start, at - start);
+}
+
 std::optional<std::string> validate_http_url(const std::string &url)
 {
     // This may be the FIRST libcurl call (config validation runs before any HttpClient is
@@ -129,9 +161,14 @@ size_t HttpClient::write_capture(char *ptr, size_t size, size_t nmemb, void *use
     size_t n = size * nmemb;
     auto *ctx = static_cast<EasyContext *>(userdata);
     if (ctx) {
-        constexpr size_t kMaxBody = 64 * 1024; // a DNS-over-HTTPS message is well under 64 KB
-        if (ctx->response.size() < kMaxBody) {
-            ctx->response.append(ptr, (n < kMaxBody - ctx->response.size()) ? n : (kMaxBody - ctx->response.size()));
+        if (ctx->response.size() + n > ctx->capture_max) {
+            // Body exceeds the cap: keep the prefix that fits and flag truncation so the caller
+            // knows the captured body is partial (a content check can't be evaluated definitively).
+            size_t room = ctx->capture_max > ctx->response.size() ? ctx->capture_max - ctx->response.size() : 0;
+            ctx->response.append(ptr, room);
+            ctx->truncated = true;
+        } else {
+            ctx->response.append(ptr, n);
         }
     }
     return n; // always consume so curl doesn't abort the transfer
@@ -174,6 +211,25 @@ void HttpClient::request(const HttpRequest &req, ResultCallback on_done)
     // We run on the netprobe io thread, not the main thread; CURLOPT_NOSIGNAL stops curl from
     // using signals (e.g. SIGALRM with the standard name resolver), which is unsafe off-main-thread.
     curl_easy_setopt(easy, CURLOPT_NOSIGNAL, 1L);
+    if (!req.user_agent.empty()) {
+        curl_easy_setopt(easy, CURLOPT_USERAGENT, req.user_agent.c_str());
+    }
+    if (!req.proxy.empty()) {
+        curl_easy_setopt(easy, CURLOPT_PROXY, req.proxy.c_str());
+        ctx->proxy = req.proxy; // retained only to redact it (and any embedded credentials) from error_msg
+    }
+    if (!req.ca_file.empty()) {
+        curl_easy_setopt(easy, CURLOPT_CAINFO, req.ca_file.c_str());
+    }
+    if (!req.cert_file.empty()) {
+        curl_easy_setopt(easy, CURLOPT_SSLCERT, req.cert_file.c_str());
+    }
+    if (!req.key_file.empty()) {
+        curl_easy_setopt(easy, CURLOPT_SSLKEY, req.key_file.c_str());
+    }
+    if (req.collect_cert_info) {
+        curl_easy_setopt(easy, CURLOPT_CERTINFO, 1L);
+    }
     if (!req.body.empty()) {
         // COPYPOSTFIELDS copies the bytes (curl owns them); size set first => binary-safe.
         curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, static_cast<long>(req.body.size()));
@@ -199,6 +255,7 @@ void HttpClient::request(const HttpRequest &req, ResultCallback on_done)
         curl_easy_setopt(easy, CURLOPT_HTTPHEADER, ctx->headers);
     }
     ctx->capture = req.capture_response;
+    ctx->capture_max = req.capture_max_bytes;
     curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, ctx->capture ? &HttpClient::write_capture : &HttpClient::write_discard);
     curl_easy_setopt(easy, CURLOPT_WRITEDATA, ctx.get());
     curl_easy_setopt(easy, CURLOPT_PRIVATE, ctx.get());
@@ -353,8 +410,29 @@ void HttpClient::check_multi_info()
             result.timings.connect_us = conn > dns ? static_cast<uint64_t>(conn - dns) : 0;
             result.timings.tls_us = app > conn ? static_cast<uint64_t>(app - conn) : 0;
             result.timings.ttfb_us = ttfb > (app ? app : conn) ? static_cast<uint64_t>(ttfb - (app ? app : conn)) : 0;
+            curl_off_t dl_size = 0;
+            curl_easy_getinfo(easy, CURLINFO_SIZE_DOWNLOAD_T, &dl_size);
+            result.response_size = dl_size > 0 ? static_cast<uint64_t>(dl_size) : 0;
+            struct curl_certinfo *ci = nullptr;
+            if (curl_easy_getinfo(easy, CURLINFO_CERTINFO, &ci) == CURLE_OK && ci) {
+                // earliest notAfter across the presented chain (blackbox_exporter semantics)
+                uint64_t earliest = 0;
+                for (int i = 0; i < ci->num_of_certs; ++i) {
+                    for (auto *sl = ci->certinfo[i]; sl; sl = sl->next) {
+                        constexpr char kPrefix[] = "Expire date:";
+                        if (sl->data && std::strncmp(sl->data, kPrefix, sizeof(kPrefix) - 1) == 0) {
+                            uint64_t e = parse_cert_expire_date(std::string(sl->data + sizeof(kPrefix) - 1));
+                            if (e && (earliest == 0 || e < earliest)) {
+                                earliest = e;
+                            }
+                        }
+                    }
+                }
+                result.cert_expiry_epoch = earliest;
+            }
             if (it != _easy.end() && it->second->capture) {
                 result.response_body = std::move(it->second->response);
+                result.body_truncated = it->second->truncated;
             }
             char *ct = nullptr;
             curl_easy_getinfo(easy, CURLINFO_CONTENT_TYPE, &ct); // may be null (no Content-Type)
@@ -371,6 +449,13 @@ void HttpClient::check_multi_info()
             } else {
                 result.error_msg = curl_easy_strerror(msg->data.result);
             }
+            // curl error text can echo the proxy URL verbatim (e.g. a malformed proxy) or its
+            // credentials; the probes log error_msg, so scrub the proxy value + userinfo here — the
+            // single choke point — to uphold the "proxy value never appears in output" guarantee.
+            if (it != _easy.end() && !it->second->proxy.empty()) {
+                redact_secret(result.error_msg, it->second->proxy);
+                redact_secret(result.error_msg, proxy_userinfo(it->second->proxy));
+            }
         }
         curl_multi_remove_handle(_multi, easy);
         curl_easy_cleanup(easy);

libs/visor_http_client/HttpClient.h

@@ -45,7 +45,10 @@ class HttpClient
         char errbuf[CURL_ERROR_SIZE]{};
         curl_slist *headers{nullptr};   // owned; freed in dtor (after curl_easy_cleanup)
         bool capture{false};
-        std::string response;           // captured body (bounded to 64 KB)
+        std::string proxy;              // configured proxy (may embed credentials); used only to REDACT it from error_msg
+        size_t capture_max{64 * 1024};  // cap on captured body bytes (from HttpRequest.capture_max_bytes)
+        bool truncated{false};          // set by write_capture when the body exceeds capture_max
+        std::string response;           // captured body (bounded to capture_max)
         ~EasyContext() { if (headers) curl_slist_free_all(headers); }
     };
     // per-socket context: a uvw poll handle curl watches (owned in _sockets below)

libs/visor_http_client/HttpTypes.h

@@ -21,14 +21,32 @@ struct HttpRequest {
     std::string body;                   // request body bytes (empty => no body)
     std::vector<std::string> headers;   // extra request headers, each "Key: Value"
     bool capture_response{false};       // when true, capture the response body
+    size_t capture_max_bytes{64 * 1024};// cap on captured response bytes; body beyond this is dropped and HttpResult.body_truncated is set
+    bool collect_cert_info{false};      // when true, request CURLOPT_CERTINFO and populate HttpResult.cert_expiry_epoch
+    std::string proxy;                  // CURLOPT_PROXY value (empty => no proxy)
+    std::string ca_file;                // CURLOPT_CAINFO (empty => curl default CA bundle)
+    std::string cert_file;              // CURLOPT_SSLCERT (client cert, empty => none)
+    std::string key_file;               // CURLOPT_SSLKEY (client key, empty => none)
+    std::string user_agent;             // CURLOPT_USERAGENT (empty => curl default)
 };
 struct HttpResult {
     bool transport_ok{false};
     long curl_code{0};
     long status_code{0};
     HttpTimings timings;
-    std::string response_body;          // populated only when HttpRequest.capture_response
+    std::string response_body;          // populated only when HttpRequest.capture_response (bounded to capture_max_bytes)
+    bool body_truncated{false};         // true when the response body exceeded capture_max_bytes (response_body is a prefix)
     std::string content_type;           // raw response Content-Type header when transport_ok (compare case-insensitively)
     std::string error_msg;              // human-readable curl error detail when !transport_ok
+    uint64_t cert_expiry_epoch{0};      // earliest "Expire date:" across the TLS chain when HttpRequest.collect_cert_info; 0 for plain http or on parse failure
+    uint64_t response_size{0};          // CURLINFO_SIZE_DOWNLOAD_T; populated on every transport_ok, independent of capture_response
+};
+struct HttpSample {
+    uint16_t status{0};
+    bool status_ok{false};        // check evaluation happens in the PROBE
+    uint8_t content_check{0};     // 0 = NotChecked, 1 = Match, 2 = Mismatch
+    uint64_t cert_expiry_epoch{0};
+    uint64_t response_size{0};
+    HttpTimings timings;
 };
 }