diff --git a/.changeset/pii-hashing-polyglot.md b/.changeset/pii-hashing-polyglot.md new file mode 100644 index 0000000..56fec97 --- /dev/null +++ b/.changeset/pii-hashing-polyglot.md @@ -0,0 +1,50 @@ +--- +'@smooai/observability': minor +--- + +TypeScript, Go, Python and .NET: hash PII instead of leaking it, matching the Rust SDK. + +All four SDKs scrubbed **credentials only** — `Bearer`, `password=`, +`token`/`api_key`/`secret=`, `sk-…` — while their module docs claimed "PII +scrubbing". Emails, phone numbers and street addresses passed through to the +backend untouched. Rust fixed this in #82; this brings the other four to parity +with byte-identical output. + +Personal identifiers are now **hashed, not dropped**: `a@b.com` → +`[email:9f2a41c8]`. The type prefix stays visible, so "are these two spans the +same person?" stays answerable while nothing reversible is stored. The hash is +**HMAC-SHA256, keyed** — not a bare digest, which a rainbow table reverses in +seconds for a space as small as email addresses — and the org id is mixed into +the message, so identical PII hashes differently in different orgs. Phone +numbers normalize to digits and emails to lowercase before hashing, so +`(415) 555-0142` and `415-555-0142` correlate. + +Credentials are still **dropped**, never hashed, and are matched first: a hash of +a live token is a token oracle, and PII inside a secret (`token=a@b.com`) goes +with the secret. With no key configured (`SMOOAI_OBSERVABILITY_PII_HASH_KEY`, or +the per-SDK setter), personal identifiers are fully redacted (`[email:redacted]`) +rather than hashed under a guessable one — fail closed, never fail open. + +New API, same shape in every SDK. The org-less entry points keep working +unchanged (they hash under the empty org salt): + +- TypeScript: `setPiiHashKey`, `piiToken`, `scrubStringForOrg`, + `scrubHeadersForOrg`, `PiiKind` — now exported from the package entry +- Go: `SetPiiHashKey`, `PiiToken`, `ScrubStringForOrg`, `ScrubHeadersForOrg`, + `PiiKind`, `BootstrapEnv.PiiHashKey` +- Python: `set_pii_hash_key`, `pii_token`, `scrub_string_for_org`, + `scrub_headers_for_org`, `PiiKind`, `BootstrapEnv.pii_hash_key` +- .NET: `Pii.SetPiiHashKey`, `Pii.PiiToken`, `Pii.ScrubStringForOrg`, + `Pii.ScrubHeadersForOrg`, `PiiKind`, `BootstrapEnv.PiiHashKey` + +`piiToken(kind, raw, orgId)` is the search seam: hash a typed query term the same +way and match the stored token. + +⚠️ **The key is load-bearing — rotate never.** Rotating it silently forks +correlation with every hash already stored. Supply it once at startup; the +setters are set-once and refuse a second key. + +The TypeScript SDK ships a small synchronous SHA-256/HMAC (`hmac-sha256.ts`) +rather than taking a dependency: `scrubString` is sync and runs in the browser +bundle, where `node:crypto` is unavailable and WebCrypto is async-only. It is +pinned by the RFC 4231 and FIPS 180-4 vectors. diff --git a/README.md b/README.md index 1de2021..21dfbb2 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ - 🗺️ **Source maps** — uploaded to S3 at build time, applied lazily on view - 🚪 **Beacon flush** — events queued at `pagehide` ship via `navigator.sendBeacon` - 💾 **Offline queue** — events captured while offline persist in `IndexedDB` and retry on focus -- 🔐 **PII scrub** — `password`, `token`, `Bearer ...`, and friends are redacted before transport +- 🔐 **PII scrub** — credentials (`password`, `token`, `Bearer ...`) are dropped; emails / phones / addresses are HMAC-hashed per-org (`a@b.com` → `[email:9f2a41c8]`) so traces stay correlatable without storing the value **Node** @@ -42,7 +42,7 @@ - 🪢 **Hono middleware** — captures errors propagating to the global `onError` handler - 🧠 **AsyncLocalStorage scope** — per-request user, tags, breadcrumbs without leaking across requests - 📦 **Batched transport** — `undici` with retry / backoff -- 🔐 **Same PII scrub policy** as the browser +- 🔐 **Same PII scrub policy** as the browser — key from `SMOOAI_OBSERVABILITY_PII_HASH_KEY` **React / Next.js** @@ -55,7 +55,8 @@ - `console.log` / `console.info` / `console.warn` — only `console.error` is tapped, and that's opt-out - HTTP request **bodies** — only method, path, status, and duration appear in breadcrumbs -- Anything matching the PII scrub regex unless you explicitly allowlist it +- Credentials matching the PII scrub regex — dropped outright, never hashed +- Raw emails / phones / street addresses — replaced by a keyed per-org hash, never stored in the clear ## 📦 Install @@ -200,7 +201,7 @@ Known divergences: TypeScript, Python, Go and .NET emit `gen_ai.tool.names` as a ## 📖 Architecture -The SDK is intentionally thin. It captures, batches, redacts PII, and POSTs to a Smoo ingest endpoint. All of the heavy lifting — fingerprint grouping, source-map symbolication, dashboards, alerts, retention — lives in the Smoo platform. +The SDK is intentionally thin. It captures, batches, redacts credentials, hashes personal identifiers, and POSTs to a Smoo ingest endpoint. All of the heavy lifting — fingerprint grouping, source-map symbolication, dashboards, alerts, retention — lives in the Smoo platform. ```mermaid %%{init: {'theme':'base','themeVariables':{ @@ -236,7 +237,7 @@ This SDK is opinionated about privacy: - We never capture form bodies, request bodies, or response bodies by default - We never capture cookies - We never send anything to a third-party service — your events go to **your** Smoo backend only -- PII scrubbing is enabled by default and can be tuned per-tenant +- PII scrubbing is enabled by default and can be tuned per-tenant. Personal identifiers are hashed with HMAC-SHA256 under a key you supply (`SMOOAI_OBSERVABILITY_PII_HASH_KEY`), salted by org id — identical across the TypeScript, Rust, Go, Python and .NET SDKs. **With no key configured they are fully redacted, never hashed under a guessable one.** ## 📖 Status diff --git a/dotnet/src/SmooAI.Observability/Bootstrap.cs b/dotnet/src/SmooAI.Observability/Bootstrap.cs index 68d7ea0..c1c7675 100644 --- a/dotnet/src/SmooAI.Observability/Bootstrap.cs +++ b/dotnet/src/SmooAI.Observability/Bootstrap.cs @@ -47,6 +47,13 @@ public sealed class BootstrapEnv /// Skip bootstrap entirely. public bool? Disabled { get; set; } + + /// + /// HMAC key used to hash emails / phones / addresses in scrubbed strings + /// (see ). Unset means personal identifiers are fully + /// redacted rather than hashed. + /// + public string? PiiHashKey { get; set; } } /// @@ -109,6 +116,11 @@ public static async Task Run(BootstrapEnv? overrides = null) var env = ResolveEnv(overrides); + // Before anything can emit: a scrubbed string written without this key + // redacts PII outright, so installing it late would silently produce a + // window of uncorrelatable spans rather than an error. + Pii.SetPiiHashKey(env.PiiHashKey); + if (env.Disabled == true) { return Cache(new BootstrapResult { Installed = false, Exporting = false, Otel = null }); @@ -242,6 +254,7 @@ internal static BootstrapEnv ResolveEnv(BootstrapEnv? overrides) Release = overrides?.Release ?? Env("SMOOAI_OBSERVABILITY_RELEASE") ?? Env("GIT_SHA") ?? Env("LAMBDA_FUNCTION_VERSION") ?? "dev", Dsn = overrides?.Dsn ?? Env("SMOOAI_OBSERVABILITY_DSN") ?? Env("OBSERVABILITY_DSN"), Disabled = overrides?.Disabled ?? Truthy(Env("SMOOAI_OBSERVABILITY_DISABLED")), + PiiHashKey = overrides?.PiiHashKey ?? Env("SMOOAI_OBSERVABILITY_PII_HASH_KEY"), }; } diff --git a/dotnet/src/SmooAI.Observability/Pii.cs b/dotnet/src/SmooAI.Observability/Pii.cs index f8c3326..fad468b 100644 --- a/dotnet/src/SmooAI.Observability/Pii.cs +++ b/dotnet/src/SmooAI.Observability/Pii.cs @@ -1,15 +1,66 @@ +using System.Security.Cryptography; +using System.Text; using System.Text.RegularExpressions; namespace SmooAI.Observability; +/// +/// The class of personal identifier a match represents. Drives both the visible +/// prefix in the output token and the normalization applied before hashing, so +/// (415) 555-0142 and 415-555-0142 correlate. +/// +public enum PiiKind +{ + /// Email address. + Email, + + /// Telephone number. + Phone, + + /// Street address. + Address, +} + /// /// PII scrubbing — applied to message strings, breadcrumb messages, and headers -/// before transport. Direct port of the TS pii.ts patterns so both SDKs -/// redact identically. Stays opinionated and minimal; tenants can extend in -/// beforeSend. +/// before transport. Port of rust/observability/src/pii.rs; the semantics +/// are identical across the five SDKs. /// +/// +/// Two classes, handled differently on purpose: +/// +/// Credentials (Bearer …, password=, +/// token/api_key/secret=, sk-…) are dropped. +/// A hash of a live token is still a token oracle, and there is no correlation +/// value in a secret. +/// Personal identifiers (email, phone, street address) +/// are hashed, not dropped: a@b.com → [email:9f2a41c8]. That +/// keeps the one question worth asking — "are these two spans the same person?" — +/// answerable while storing nothing reversible. +/// +/// +/// The hash is HMAC-SHA256, keyed, not a bare digest: emails and phone +/// numbers are a small enumerable space that a rainbow table reverses in seconds. +/// The org id is mixed into the HMAC message, so identical PII hashes +/// differently in different orgs. +/// +/// +/// The key and the org salt are load-bearing and must not rotate casually. +/// Rotating either silently breaks correlation with every previously stored hash. +/// Supply the key once at startup via SMOOAI_OBSERVABILITY_PII_HASH_KEY +/// (read by ) or . +/// With no key configured, personal identifiers are fully redacted +/// ([email:redacted]) rather than hashed under a guessable one — fail safe, +/// never fail open. +/// +/// public static partial class Pii { + // ---- credentials: matched FIRST, dropped entirely ---------------------- + // + // A personal identifier sitting inside a secret (token=a@b.com) is dropped + // with the secret rather than surviving as a hash. + // Bearer tokens. [GeneratedRegex(@"Bearer\s+[A-Za-z0-9._-]+", RegexOptions.IgnoreCase)] private static partial Regex BearerRegex(); @@ -26,6 +77,29 @@ public static partial class Pii [GeneratedRegex(@"\bsk-[A-Za-z0-9]{20,}")] private static partial Regex SkKeyRegex(); + // ---- personal identifiers: hashed, not dropped ------------------------- + + [GeneratedRegex(@"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+\b", RegexOptions.IgnoreCase)] + private static partial Regex EmailRegex(); + + // Phone: optional country code / area code, then the 3-4 local pair. A + // separator is REQUIRED so bare digit runs (ids, timestamps, amounts) don't + // get eaten. + [GeneratedRegex(@"(?:\+\d{1,3}[ .-]?)?(?:\(\d{3}\)[ .-]?|\b\d{3}[ .-])?\b\d{3}[ .-]\d{4}\b")] + private static partial Regex PhoneRegex(); + + // US-style street address: house number, 1-3 words, a street suffix. + [GeneratedRegex( + @"\b\d{1,6}\s+(?:[A-Za-z0-9.'-]+\s+){0,3}(?:street|st|avenue|ave|road|rd|boulevard|blvd|lane|ln|drive|dr|court|ct|way|terrace|ter|place|pl|circle|cir|highway|hwy|parkway|pkwy|square|sq)\b\.?", + RegexOptions.IgnoreCase)] + private static partial Regex AddressRegex(); + + [GeneratedRegex(@"[^0-9]")] + private static partial Regex NonDigitRegex(); + + [GeneratedRegex(@"\s+")] + private static partial Regex WhitespaceRunRegex(); + private static readonly HashSet SensitiveHeaders = new(StringComparer.OrdinalIgnoreCase) { "authorization", @@ -36,32 +110,72 @@ public static partial class Pii }; /// - /// Redact known sensitive substrings from a free-form string. + /// Hex characters kept from the HMAC. Long enough that collisions are rare + /// across an org's traces, short enough to read in a span attribute. + /// + internal const int HashHexLen = 8; + + private static byte[]? _piiHashKey; + + /// + /// Install the process-wide HMAC key used to hash personal identifiers. + /// Idempotent and set-once: returns false if a key was already + /// installed (the existing key is kept — a mid-process rotation would + /// silently fork every correlation) or if is empty. + /// calls this from + /// SMOOAI_OBSERVABILITY_PII_HASH_KEY. /// - /// - /// The token/apikey/secret pattern intentionally redacts only the value after - /// the delimiter (replacing key=value with key=[redacted]), - /// matching the TS port's effective behavior. - /// - public static string ScrubString(string? input) + public static bool SetPiiHashKey(byte[]? key) { - if (string.IsNullOrEmpty(input)) + if (key is null || key.Length == 0) { - return input ?? string.Empty; + return false; } - - var output = BearerRegex().Replace(input, "Bearer [redacted]"); - output = PasswordRegex().Replace(output, "password=[redacted]"); - output = SecretRegex().Replace(output, RedactAfterDelimiter); - output = SkKeyRegex().Replace(output, "sk-[redacted]"); - return output; + return Interlocked.CompareExchange(ref _piiHashKey, (byte[])key.Clone(), null) is null; } + /// + /// from a UTF-8 string. + /// + public static bool SetPiiHashKey(string? key) => + !string.IsNullOrEmpty(key) && SetPiiHashKey(Encoding.UTF8.GetBytes(key)); + + /// + /// Hash one known-personal value into its scrubbed token — the same token + /// would have written. This is how a UI + /// search box finds stored hashes: hash the typed term with the same org and + /// match. Returns [<kind>:redacted] when no key is installed. + /// + public static string PiiToken(PiiKind kind, string raw, string orgId) => + TokenWithKey(kind, raw, orgId, Volatile.Read(ref _piiHashKey)); + + /// + /// Scrub a free-form string with no org context — credentials dropped, + /// personal identifiers hashed under the empty org salt. Prefer + /// wherever an org id is in hand, so hashes + /// can't be correlated across tenants. + /// + public static string ScrubString(string? input) => ScrubWithKey(input, string.Empty, Volatile.Read(ref _piiHashKey)); + + /// + /// Scrub a free-form string, salting personal-identifier hashes with + /// . + /// + public static string ScrubStringForOrg(string? input, string orgId) => + ScrubWithKey(input, orgId, Volatile.Read(ref _piiHashKey)); + /// /// Scrub a header dictionary: sensitive header names are fully redacted, all /// other values are passed through . /// - public static Dictionary? ScrubHeaders(IReadOnlyDictionary? headers) + public static Dictionary? ScrubHeaders(IReadOnlyDictionary? headers) => + ScrubHeadersForOrg(headers, string.Empty); + + /// + /// with an org salt for the personal-identifier + /// hashes. + /// + public static Dictionary? ScrubHeadersForOrg(IReadOnlyDictionary? headers, string orgId) { if (headers is null) { @@ -71,17 +185,83 @@ public static string ScrubString(string? input) var output = new Dictionary(headers.Count, StringComparer.Ordinal); foreach (var (key, value) in headers) { - output[key] = SensitiveHeaders.Contains(key) ? "[redacted]" : ScrubString(value); + output[key] = SensitiveHeaders.Contains(key) ? "[redacted]" : ScrubStringForOrg(value, orgId); } return output; } + // Test seam: drives the scrub with an explicit key so the suite doesn't have + // to write the set-once process-wide key (xUnit runs the class in one + // process, which would make a global write order-dependent). + internal static string ScrubWithKey(string? input, string orgId, byte[]? key) + { + if (string.IsNullOrEmpty(input)) + { + return input ?? string.Empty; + } + + // Credentials first — dropped, never hashed. + var output = BearerRegex().Replace(input, "Bearer [redacted]"); + output = PasswordRegex().Replace(output, "password=[redacted]"); + output = SecretRegex().Replace(output, RedactAfterDelimiter); + output = SkKeyRegex().Replace(output, "sk-[redacted]"); + + // Personal identifiers — hashed, prefix preserved. + output = EmailRegex().Replace(output, m => TokenWithKey(PiiKind.Email, m.Value, orgId, key)); + output = PhoneRegex().Replace(output, m => TokenWithKey(PiiKind.Phone, m.Value, orgId, key)); + output = AddressRegex().Replace(output, m => TokenWithKey(PiiKind.Address, m.Value, orgId, key)); + return output; + } + + internal static string TokenWithKey(PiiKind kind, string raw, string orgId, byte[]? key) + { + var label = Label(kind); + if (key is null || key.Length == 0) + { + return $"[{label}:redacted]"; + } + + // orgId in the HMAC message IS the per-org salt. The kind is in there too + // so a phone and an address that normalize alike can't collide. + var message = new List(); + message.AddRange(Encoding.UTF8.GetBytes(orgId)); + message.Add(0); + message.AddRange(Encoding.UTF8.GetBytes(label)); + message.Add(0); + message.AddRange(Encoding.UTF8.GetBytes(Normalize(kind, raw))); + + var mac = HMACSHA256.HashData(key, message.ToArray()); + var hex = Convert.ToHexString(mac, 0, HashHexLen / 2).ToLowerInvariant(); + return $"[{label}:{hex}]"; + } + + // Test seam: clears the set-once key so the set-once behavior itself can be + // asserted. Not part of the public surface — rotation is never valid at runtime. + internal static void ResetPiiHashKeyForTests() => Volatile.Write(ref _piiHashKey, null); + + /// The prefix that stays visible in the scrubbed output. + internal static string Label(PiiKind kind) => kind switch + { + PiiKind.Email => "email", + PiiKind.Phone => "phone", + _ => "address", + }; + + private static string Normalize(PiiKind kind, string raw) => kind switch + { + // Digits only: formatting must not fork the hash. + PiiKind.Phone => NonDigitRegex().Replace(raw, string.Empty), + PiiKind.Email => raw.Trim().ToLowerInvariant(), + // Case-fold and collapse runs of whitespace. + _ => WhitespaceRunRegex().Replace(raw.Trim(), " ").ToLowerInvariant(), + }; + // Replace everything from the first delimiter onward with =[redacted], keeping // the key prefix intact (e.g. "token: abc" -> "token=[redacted]"). private static string RedactAfterDelimiter(Match match) { var value = match.Value; - var delimiterIndex = value.IndexOfAny(new[] { ':', '=' }); + var delimiterIndex = value.IndexOfAny([':', '=']); if (delimiterIndex < 0) { return value; diff --git a/dotnet/src/SmooAI.Observability/README.md b/dotnet/src/SmooAI.Observability/README.md index d863713..f15689b 100644 --- a/dotnet/src/SmooAI.Observability/README.md +++ b/dotnet/src/SmooAI.Observability/README.md @@ -116,6 +116,7 @@ GenAIActivity.SetAttributes(Activity.Current, new GenAIAttributes | `SMOOAI_OBSERVABILITY_RELEASE` | Release id | | `SMOOAI_OBSERVABILITY_DSN` | Error-webhook DSN (optional) | | `SMOOAI_OBSERVABILITY_DISABLED` | `1`/`true` to skip bootstrap | +| `SMOOAI_OBSERVABILITY_PII_HASH_KEY` | HMAC key for hashing emails / phones / addresses (unset = redacted) | ## Design notes @@ -128,3 +129,30 @@ GenAIActivity.SetAttributes(Activity.Current, new GenAIAttributes SDK (`System.Text.Json`, nulls omitted). Tracking: [SMOODEV-1159](https://smooai.atlassian.net/browse/SMOODEV-1159). + +## PII scrubbing + +Two classes, handled differently: + +- **Credentials** (`Bearer …`, `password=`, `token`/`api_key`/`secret=`, `sk-…`) + are **dropped**. A hash of a live token is still a token oracle. +- **Personal identifiers** (email, phone, street address) are **hashed**: + `a@b.com` → `[email:9f2a41c8]`. The type prefix stays visible, so you can see + *what kind* of value was there and that two spans carry the *same* one — + without ever seeing it. + +The hash is **HMAC-SHA256**, not a bare digest (emails and phones are a small +enumerable space a rainbow table reverses in seconds), and the org id is mixed +into the message so the same value hashes **differently in different orgs**. + +Set the key with `SMOOAI_OBSERVABILITY_PII_HASH_KEY` (read by the bootstrap) or +`Pii.SetPiiHashKey(...)`. **With no key, personal identifiers are fully redacted** +(`[email:redacted]`) rather than hashed under a guessable one. + +⚠️ **The key and the org id are load-bearing.** Rotating either silently breaks +correlation with every hash already stored — treat the key as permanent, and do +not reuse a secret that rotates on a schedule. + +All five SDKs (TypeScript, Rust, Go, Python, .NET) emit byte-identical tokens +for the same key/org/value; the shared vectors are asserted in each SDK's PII +test suite. diff --git a/dotnet/tests/SmooAI.Observability.Tests/PiiTests.cs b/dotnet/tests/SmooAI.Observability.Tests/PiiTests.cs index d27e66e..b3d0ec1 100644 --- a/dotnet/tests/SmooAI.Observability.Tests/PiiTests.cs +++ b/dotnet/tests/SmooAI.Observability.Tests/PiiTests.cs @@ -87,3 +87,193 @@ public void ScrubHeaders_ReturnsNullForNull() Assert.Null(Pii.ScrubHeaders(null)); } } + +/// +/// PII hashing parity with rust/observability/src/pii.rs. +/// +/// +/// These drive the internal ScrubWithKey / TokenWithKey rather than +/// installing the process-wide key: SetPiiHashKey is set-once and xUnit +/// runs the assembly in one process, so a global write would make the suite +/// order-dependent. +/// +public class PiiHashingTests +{ + private static readonly byte[] Key = "test-hmac-key-not-a-real-secret"u8.ToArray(); + private static readonly byte[] OtherKey = "a-different-test-hmac-key"u8.ToArray(); + + private static string Scrub(string input, string org = "org-1") => Pii.ScrubWithKey(input, org, Key); + + [Fact] + public void CredentialsAreDroppedNotHashed() + { + // A live token must never become a correlatable handle — hashing one still + // yields an oracle you can test candidate tokens against. + var result = Scrub("Bearer abc.def-ghi_123 password=hunter2 sk-ABCDEFGHIJKLMNOPQRSTUVWX"); + Assert.Contains("Bearer [redacted]", result); + Assert.Contains("password=[redacted]", result); + Assert.Contains("sk-[redacted]", result); + Assert.DoesNotContain("[token:", result); + Assert.DoesNotContain("[credential:", result); + Assert.DoesNotContain("[email:", result); + Assert.DoesNotContain("[phone:", result); + + // And a personal identifier hiding inside a secret goes with it. + var inSecret = Scrub("token=a@b.com"); + Assert.DoesNotContain("a@b.com", inSecret); + Assert.DoesNotContain("[email:", inSecret); + } + + [Fact] + public void HashesEmailsKeepingTheTypePrefix() + { + var result = Scrub("contact me at Alice@Example.com please"); + Assert.DoesNotContain("alice@example.com", result.ToLowerInvariant()); + Assert.StartsWith("contact me at [email:", result); + Assert.EndsWith("] please", result); + } + + [Theory] + [InlineData("555-0142")] + [InlineData("(415) 555-0142")] + [InlineData("+1 415-555-0142")] + public void HashesPhoneNumbers(string raw) + { + var result = Scrub($"call {raw} today"); + Assert.Contains("[phone:", result); + Assert.DoesNotContain("0142", result); + } + + [Fact] + public void HashesStreetAddresses() + { + var result = Scrub("ship to 1600 Pennsylvania Ave, Washington"); + Assert.Contains("[address:", result); + Assert.DoesNotContain("Pennsylvania", result); + } + + [Fact] + public void SameValueSameOrgIsStable() + { + Assert.Equal(Scrub("a@b.com"), Scrub("a@b.com")); + // …and correlation survives formatting differences in phones. + Assert.Equal(Scrub("(415) 555-0142"), Scrub("415-555-0142")); + } + + [Fact] + public void SameValueDifferentOrgHashesDifferently() + { + var a = Scrub("a@b.com", "org-1"); + var b = Scrub("a@b.com", "org-2"); + Assert.NotEqual(a, b); + Assert.StartsWith("[email:", a); + Assert.StartsWith("[email:", b); + } + + [Fact] + public void DifferentKeyHashesDifferently() + { + Assert.NotEqual( + Pii.ScrubWithKey("a@b.com", "org-1", Key), + Pii.ScrubWithKey("a@b.com", "org-1", OtherKey)); + } + + [Fact] + public void NoKeyRedactsRatherThanHashing() + { + // Fail safe: an unkeyed digest of an email is rainbow-tabled instantly. + var result = Pii.ScrubWithKey("a@b.com and 555-0142", "org-1", null); + Assert.Contains("[email:redacted]", result); + Assert.Contains("[phone:redacted]", result); + Assert.DoesNotContain("a@b.com", result); + Assert.DoesNotContain("0142", result); + } + + [Fact] + public void HashIsShortHex() + { + var token = Pii.TokenWithKey(PiiKind.Email, "a@b.com", "org-1", Key); + var hex = token["[email:".Length..^1]; + Assert.Equal(Pii.HashHexLen, hex.Length); + Assert.All(hex, c => Assert.Contains(c, "0123456789abcdef")); + } + + [Fact] + public void PiiTokenMatchesWhatScrubbingWrote() + { + // The searchability contract: hashing a typed query term must produce + // exactly the token stored in the span. + var scrubbed = Pii.ScrubWithKey("mail a@b.com now", "org-7", Key); + var term = Pii.TokenWithKey(PiiKind.Email, "A@B.com ", "org-7", Key); + Assert.Contains(term, scrubbed); + } + + [Theory] + [InlineData("took 1234 ms")] + [InlineData("version 1.2.3")] + [InlineData("2026-08-15T12:00:00Z")] + [InlineData("total 1234.5678")] + [InlineData("trace 0af7651916cd43dd8448eb211c80319c")] + public void OrdinaryNumbersAreNotMistakenForPhones(string value) + { + // The phone pattern requires a separator before the last 4 digits; ids, + // versions, dates and amounts must survive intact. + Assert.Equal(value, Scrub(value)); + } + + [Fact] + public void SetPiiHashKeyIsSetOnceAndRejectsEmpty() + { + Assert.False(Pii.SetPiiHashKey((byte[]?)null)); + Assert.False(Pii.SetPiiHashKey(Array.Empty())); + Assert.False(Pii.SetPiiHashKey((string?)null)); + + Pii.ResetPiiHashKeyForTests(); + try + { + Assert.True(Pii.SetPiiHashKey("first-key")); + Assert.False(Pii.SetPiiHashKey("second-key")); + // The first key is what is still in force. + Assert.Equal( + Pii.TokenWithKey(PiiKind.Email, "a@b.com", "o", "first-key"u8.ToArray()), + Pii.PiiToken(PiiKind.Email, "a@b.com", "o")); + } + finally + { + Pii.ResetPiiHashKeyForTests(); + } + } + + [Fact] + public void ScrubHeadersForOrgNeverLeaksRawEmail() + { + var headers = new Dictionary { ["X-Note"] = "reply to a@b.com" }; + var scrubbed = Pii.ScrubHeadersForOrg(headers, "org-1")!; + Assert.DoesNotContain("a@b.com", scrubbed["X-Note"]); + Assert.Contains("[email:", scrubbed["X-Note"]); + } +} + +/// +/// Pins the exact bytes every SDK must produce. Computed independently +/// (python hmac.new(key, org\0kind\0normalized, "sha256")) and asserted +/// verbatim in all five SDKs. If any SDK's message framing, normalization or +/// truncation drifts, exactly one of these breaks. +/// +public class PiiCrossSdkParityTests +{ + private static readonly byte[] Key = "test-hmac-key-not-a-real-secret"u8.ToArray(); + + [Theory] + [InlineData(PiiKind.Email, "a@b.com", "org-1", "[email:02ea437f]")] + [InlineData(PiiKind.Email, "A@B.COM ", "org-1", "[email:02ea437f]")] + [InlineData(PiiKind.Email, "a@b.com", "org-2", "[email:fd96f7dc]")] + [InlineData(PiiKind.Email, "a@b.com", "", "[email:453b154f]")] + [InlineData(PiiKind.Phone, "(415) 555-0142", "org-1", "[phone:415a9aea]")] + [InlineData(PiiKind.Phone, "415-555-0142", "org-1", "[phone:415a9aea]")] + [InlineData(PiiKind.Address, "1600 Pennsylvania Ave", "org-1", "[address:c5351f4a]")] + public void MatchesTheSharedVectors(PiiKind kind, string raw, string orgId, string expected) + { + Assert.Equal(expected, Pii.TokenWithKey(kind, raw, orgId, Key)); + } +} diff --git a/go/README.md b/go/README.md index 0877405..6167561 100644 --- a/go/README.md +++ b/go/README.md @@ -202,3 +202,30 @@ so one backend ingest endpoint (`type: "error"`) serves both SDKs. The SDK name The plain `CaptureException` still records via transport + a synthetic span. - **`sendBeacon` / page-unload** — browser-only; N/A for Go. - **Source-map / symbolication** — Go stacks are already symbolicated. + +## PII scrubbing + +Two classes, handled differently: + +- **Credentials** (`Bearer …`, `password=`, `token`/`api_key`/`secret=`, `sk-…`) + are **dropped**. A hash of a live token is still a token oracle. +- **Personal identifiers** (email, phone, street address) are **hashed**: + `a@b.com` → `[email:9f2a41c8]`. The type prefix stays visible, so you can see + *what kind* of value was there and that two spans carry the *same* one — + without ever seeing it. + +The hash is **HMAC-SHA256**, not a bare digest (emails and phones are a small +enumerable space a rainbow table reverses in seconds), and the org id is mixed +into the message so the same value hashes **differently in different orgs**. + +Set the key with `SMOOAI_OBSERVABILITY_PII_HASH_KEY` (read by the bootstrap) or +`observability.SetPiiHashKey(...)`. **With no key, personal identifiers are fully redacted** +(`[email:redacted]`) rather than hashed under a guessable one. + +⚠️ **The key and the org id are load-bearing.** Rotating either silently breaks +correlation with every hash already stored — treat the key as permanent, and do +not reuse a secret that rotates on a schedule. + +All five SDKs (TypeScript, Rust, Go, Python, .NET) emit byte-identical tokens +for the same key/org/value; the shared vectors are asserted in each SDK's PII +test suite. diff --git a/go/bootstrap.go b/go/bootstrap.go index 421dbbf..17516ab 100644 --- a/go/bootstrap.go +++ b/go/bootstrap.go @@ -23,7 +23,8 @@ import ( // // Optional: _SERVICE_NAME (default "smoo-service"), _ENVIRONMENT // (default STAGE / GO_ENV / "unknown"), _RELEASE (default GIT_SHA / "dev"), -// _DISABLED ("1"/"true" to skip). +// _DISABLED ("1"/"true" to skip), _PII_HASH_KEY (HMAC key for hashing emails / +// phones / addresses — unset means they are fully redacted; see pii.go). // BootstrapResult reports what the bootstrap did. type BootstrapResult struct { @@ -61,6 +62,11 @@ type BootstrapEnv struct { // to both OTel and the Errors dashboard (SMOODEV-1148 parity). Resolved // from OBSERVABILITY_DSN when not overridden. DSN string + // PiiHashKey is the HMAC key used to hash emails / phones / addresses in + // scrubbed strings (see pii.go). Unset means personal identifiers are fully + // redacted rather than hashed. Resolved from + // SMOOAI_OBSERVABILITY_PII_HASH_KEY. + PiiHashKey string } var ( @@ -88,6 +94,13 @@ func Bootstrap(ctx context.Context, overrides *BootstrapEnv) BootstrapResult { env := resolveEnv(overrides) + // Before anything can emit: a scrubbed string written without this key + // redacts PII outright, so installing it late would silently produce a + // window of uncorrelatable spans rather than an error. + if env.PiiHashKey != "" { + SetPiiHashKey([]byte(env.PiiHashKey)) + } + if env.Disabled { result = BootstrapResult{Installed: false} bootstrapResult = &result @@ -211,6 +224,7 @@ func resolveEnv(o *BootstrapEnv) BootstrapEnv { Release: firstNonEmpty(o.Release, os.Getenv("SMOOAI_OBSERVABILITY_RELEASE"), os.Getenv("GIT_SHA"), "dev"), Disabled: o.Disabled || truthy(os.Getenv("SMOOAI_OBSERVABILITY_DISABLED")), DSN: firstNonEmpty(o.DSN, os.Getenv("OBSERVABILITY_DSN")), + PiiHashKey: pick(o.PiiHashKey, "SMOOAI_OBSERVABILITY_PII_HASH_KEY"), } } diff --git a/go/bootstrap_test.go b/go/bootstrap_test.go index 6d42161..f00b7dc 100644 --- a/go/bootstrap_test.go +++ b/go/bootstrap_test.go @@ -158,3 +158,14 @@ func TestBootstrapExportingWhenEndpointConfigured(t *testing.T) { t.Error("expected a tracer provider when an endpoint is configured") } } + +func TestResolveEnvReadsPiiHashKey(t *testing.T) { + t.Setenv("SMOOAI_OBSERVABILITY_PII_HASH_KEY", "env-supplied-key") + if got := resolveEnv(nil).PiiHashKey; got != "env-supplied-key" { + t.Errorf("PiiHashKey = %q, want the env value", got) + } + // An explicit override still wins. + if got := resolveEnv(&BootstrapEnv{PiiHashKey: "override"}).PiiHashKey; got != "override" { + t.Errorf("override ignored: %q", got) + } +} diff --git a/go/pii.go b/go/pii.go index 970880a..36cfe02 100644 --- a/go/pii.go +++ b/go/pii.go @@ -1,14 +1,39 @@ package observability import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" "regexp" "strings" + "sync" ) // PII scrubbing — applied to message strings, breadcrumb messages, and headers -// before transport. Ports the exact patterns from the TS reference SDK -// (packages/core/src/pii.ts). Stays opinionated and minimal; callers can extend -// in a BeforeSend hook. +// before transport. Port of rust/observability/src/pii.rs; the semantics are +// identical across the five SDKs. +// +// Two classes, handled differently on purpose: +// +// - Credentials (Bearer …, password=, token/api_key/secret=, sk-…) are +// DROPPED. A hash of a live token is still a token oracle, and there is no +// correlation value in a secret. +// - Personal identifiers (email, phone, street address) are HASHED, not +// dropped: a@b.com -> [email:9f2a41c8]. That keeps the one question worth +// asking — "are these two spans the same person?" — answerable while +// storing nothing reversible. +// +// The hash is HMAC-SHA256, keyed, not a bare digest: emails and phone numbers +// are a small enumerable space that a rainbow table reverses in seconds. The +// org id is mixed into the HMAC message, so identical PII hashes DIFFERENTLY in +// different orgs. +// +// The key and the org salt are load-bearing and must not rotate casually. +// Rotating either silently breaks correlation with every previously stored +// hash. Supply the key once at startup via SMOOAI_OBSERVABILITY_PII_HASH_KEY +// (read by Bootstrap) or SetPiiHashKey. With no key configured, personal +// identifiers are fully redacted ([email:redacted]) rather than hashed under a +// guessable one — fail safe, never fail open. // Go's regexp (RE2) has no backreferences, so the TS pattern that reused `$&` // with a JS `.replace` callback is reimplemented with a capture group + @@ -19,7 +44,10 @@ type piiPattern struct { replacement string } -var piiPatterns = []piiPattern{ +// credentialPatterns are matched FIRST so a personal identifier sitting inside +// a secret (token=a@b.com) is dropped with the secret rather than surviving as +// a hash. +var credentialPatterns = []piiPattern{ // Bearer tokens. {regexp.MustCompile(`(?i)Bearer\s+[A-Za-z0-9._-]+`), "Bearer [redacted]"}, // password=... / passwd: ... / pwd = ... @@ -32,6 +60,52 @@ var piiPatterns = []piiPattern{ {regexp.MustCompile(`sk-[A-Za-z0-9]{20,}`), "sk-[redacted]"}, } +// PiiKind is the class of personal identifier a match represents. It drives both +// the visible prefix in the output token and the normalization applied before +// hashing, so "(415) 555-0142" and "415-555-0142" correlate. +type PiiKind string + +const ( + PiiEmail PiiKind = "email" + PiiPhone PiiKind = "phone" + PiiAddress PiiKind = "address" +) + +// Label is the prefix that stays visible in the scrubbed output. +func (k PiiKind) Label() string { return string(k) } + +var nonDigit = regexp.MustCompile(`[^0-9]`) + +func (k PiiKind) normalize(raw string) string { + switch k { + case PiiEmail: + return strings.ToLower(strings.TrimSpace(raw)) + case PiiPhone: + // Digits only: formatting must not fork the hash. + return nonDigit.ReplaceAllString(raw, "") + case PiiAddress: + // Case-fold and collapse runs of whitespace. + return strings.ToLower(strings.Join(strings.Fields(raw), " ")) + default: + return raw + } +} + +// personalPatterns are hashed, not dropped. Order matters only in that these +// all run after credentialPatterns. +var personalPatterns = []struct { + kind PiiKind + re *regexp.Regexp +}{ + {PiiEmail, regexp.MustCompile(`(?i)\b[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+\b`)}, + // Phone: optional country code / area code, then the 3-4 local pair. A + // separator is REQUIRED so bare digit runs (ids, timestamps, amounts) don't + // get eaten. + {PiiPhone, regexp.MustCompile(`(?:\+\d{1,3}[ .-]?)?(?:\(\d{3}\)[ .-]?|\b\d{3}[ .-])?\b\d{3}[ .-]\d{4}\b`)}, + // US-style street address: house number, 1-3 words, a street suffix. + {PiiAddress, regexp.MustCompile(`(?i)\b\d{1,6}\s+(?:[A-Za-z0-9.'-]+\s+){0,3}(?:street|st|avenue|ave|road|rd|boulevard|blvd|lane|ln|drive|dr|court|ct|way|terrace|ter|place|pl|circle|cir|highway|hwy|parkway|pkwy|square|sq)\b\.?`)}, +} + // sensitiveHeaders are header names whose values are fully redacted. var sensitiveHeaders = map[string]struct{}{ "authorization": {}, @@ -41,18 +115,101 @@ var sensitiveHeaders = map[string]struct{}{ "x-auth-token": {}, } -// ScrubString applies the PII patterns to a single string. +// hashHexLen is how many hex characters are kept from the HMAC. Long enough +// that collisions are rare across an org's traces, short enough to read in a +// span attribute. +const hashHexLen = 8 + +var ( + piiKeyMu sync.RWMutex + piiKey []byte +) + +// SetPiiHashKey installs the process-wide HMAC key used to hash personal +// identifiers. Idempotent and set-once: returns false if a key was already +// installed (the existing key is kept — a mid-process rotation would silently +// fork every correlation) or if key is empty. Bootstrap calls this from +// SMOOAI_OBSERVABILITY_PII_HASH_KEY. +func SetPiiHashKey(key []byte) bool { + if len(key) == 0 { + return false + } + piiKeyMu.Lock() + defer piiKeyMu.Unlock() + if piiKey != nil { + return false + } + piiKey = append([]byte(nil), key...) + return true +} + +func piiHashKey() []byte { + piiKeyMu.RLock() + defer piiKeyMu.RUnlock() + return piiKey +} + +// PiiToken hashes one known-personal value into its scrubbed token — the same +// token ScrubStringForOrg would have written. This is how a UI search box finds +// stored hashes: hash the typed term with the same org and match. +// +// Returns "[:redacted]" when no key is installed. +func PiiToken(kind PiiKind, raw, orgID string) string { + return tokenWithKey(kind, raw, orgID, piiHashKey()) +} + +func tokenWithKey(kind PiiKind, raw, orgID string, key []byte) string { + if len(key) == 0 { + return "[" + kind.Label() + ":redacted]" + } + // orgID in the HMAC message IS the per-org salt. The kind is in there too so + // a phone and an address that normalize alike can't collide. + mac := hmac.New(sha256.New, key) + mac.Write([]byte(orgID)) + mac.Write([]byte{0}) + mac.Write([]byte(kind.Label())) + mac.Write([]byte{0}) + mac.Write([]byte(kind.normalize(raw))) + return "[" + kind.Label() + ":" + hex.EncodeToString(mac.Sum(nil))[:hashHexLen] + "]" +} + +// ScrubString scrubs a free-form string with no org context — credentials +// dropped, personal identifiers hashed under the empty org salt. Prefer +// ScrubStringForOrg wherever an org id is in hand, so hashes can't be +// correlated across tenants. func ScrubString(input string) string { + return scrubWithKey(input, "", piiHashKey()) +} + +// ScrubStringForOrg scrubs a free-form string, salting personal-identifier +// hashes with orgID. +func ScrubStringForOrg(input, orgID string) string { + return scrubWithKey(input, orgID, piiHashKey()) +} + +func scrubWithKey(input, orgID string, key []byte) string { out := input - for _, p := range piiPatterns { + for _, p := range credentialPatterns { out = p.re.ReplaceAllString(out, p.replacement) } + for _, p := range personalPatterns { + kind := p.kind + out = p.re.ReplaceAllStringFunc(out, func(match string) string { + return tokenWithKey(kind, match, orgID, key) + }) + } return out } // ScrubHeaders fully redacts sensitive header values and scrubs the rest. // Returns nil for a nil map (matches the TS undefined passthrough). func ScrubHeaders(headers map[string]string) map[string]string { + return ScrubHeadersForOrg(headers, "") +} + +// ScrubHeadersForOrg is ScrubHeaders with an org salt for the +// personal-identifier hashes. +func ScrubHeadersForOrg(headers map[string]string, orgID string) map[string]string { if headers == nil { return nil } @@ -61,7 +218,7 @@ func ScrubHeaders(headers map[string]string) map[string]string { if _, ok := sensitiveHeaders[strings.ToLower(k)]; ok { out[k] = "[redacted]" } else { - out[k] = ScrubString(v) + out[k] = ScrubStringForOrg(v, orgID) } } return out diff --git a/go/pii_test.go b/go/pii_test.go index 6a1ebe6..4447700 100644 --- a/go/pii_test.go +++ b/go/pii_test.go @@ -1,6 +1,9 @@ package observability -import "testing" +import ( + "strings" + "testing" +) func TestScrubString(t *testing.T) { cases := []struct { @@ -55,3 +58,210 @@ func TestScrubHeaders(t *testing.T) { t.Errorf("Content-Type mangled: %q", out["Content-Type"]) } } + +// ---- PII hashing parity with rust/observability/src/pii.rs ----------------- +// +// These drive scrubWithKey / tokenWithKey rather than installing the +// process-wide key: SetPiiHashKey is set-once and `go test` runs the package in +// one process, so a global write would make the suite order-dependent. + +const ( + testKey = "test-hmac-key-not-a-real-secret" + testOtherKey = "a-different-test-hmac-key" +) + +func scrubT(input, org string) string { + return scrubWithKey(input, org, []byte(testKey)) +} + +func TestCredentialsAreDroppedNotHashed(t *testing.T) { + // A live token must never become a correlatable handle — hashing one still + // yields an oracle you can test candidate tokens against. + s := scrubT("Bearer abc.def-ghi_123 password=hunter2 sk-ABCDEFGHIJKLMNOPQRSTUVWX", "org-1") + for _, want := range []string{"Bearer [redacted]", "password=[redacted]", "sk-[redacted]"} { + if !strings.Contains(s, want) { + t.Errorf("missing %q in %q", want, s) + } + } + for _, bad := range []string{"[token:", "[credential:", "[email:", "[phone:"} { + if strings.Contains(s, bad) { + t.Errorf("credential was hashed (%q) in %q", bad, s) + } + } + // And a personal identifier hiding inside a secret goes with it. + s2 := scrubT("token=a@b.com", "org-1") + if strings.Contains(s2, "a@b.com") || strings.Contains(s2, "[email:") { + t.Errorf("PII inside a secret survived: %q", s2) + } +} + +func TestHashesEmailsKeepingTheTypePrefix(t *testing.T) { + s := scrubT("contact me at Alice@Example.com please", "org-1") + if strings.Contains(strings.ToLower(s), "alice@example.com") { + t.Fatalf("raw email survived: %q", s) + } + if !strings.HasPrefix(s, "contact me at [email:") || !strings.HasSuffix(s, "] please") { + t.Errorf("type prefix not preserved in place: %q", s) + } +} + +func TestHashesPhoneNumbers(t *testing.T) { + for _, raw := range []string{"555-0142", "(415) 555-0142", "+1 415-555-0142"} { + s := scrubT("call "+raw+" today", "org-1") + if !strings.Contains(s, "[phone:") { + t.Errorf("%q not hashed: %q", raw, s) + } + if strings.Contains(s, "0142") { + t.Errorf("%q leaked digits: %q", raw, s) + } + } +} + +func TestHashesStreetAddresses(t *testing.T) { + s := scrubT("ship to 1600 Pennsylvania Ave, Washington", "org-1") + if !strings.Contains(s, "[address:") || strings.Contains(s, "Pennsylvania") { + t.Errorf("address not hashed: %q", s) + } +} + +func TestSameValueSameOrgIsStable(t *testing.T) { + if a, b := scrubT("a@b.com", "org-1"), scrubT("a@b.com", "org-1"); a != b { + t.Errorf("not deterministic: %q != %q", a, b) + } + // …and correlation survives formatting differences in phones. + if a, b := scrubT("(415) 555-0142", "org-1"), scrubT("415-555-0142", "org-1"); a != b { + t.Errorf("normalization missing: %q != %q", a, b) + } +} + +func TestSameValueDifferentOrgHashesDifferently(t *testing.T) { + a, b := scrubT("a@b.com", "org-1"), scrubT("a@b.com", "org-2") + if a == b { + t.Errorf("per-org salt missing: %s == %s", a, b) + } + if !strings.HasPrefix(a, "[email:") || !strings.HasPrefix(b, "[email:") { + t.Errorf("prefix lost: %q %q", a, b) + } +} + +func TestDifferentKeyHashesDifferently(t *testing.T) { + a := scrubWithKey("a@b.com", "org-1", []byte(testKey)) + b := scrubWithKey("a@b.com", "org-1", []byte(testOtherKey)) + if a == b { + t.Errorf("hash is not keyed: %s == %s", a, b) + } +} + +func TestNoKeyRedactsRatherThanHashing(t *testing.T) { + // Fail safe: an unkeyed digest of an email is rainbow-tabled instantly. + s := scrubWithKey("a@b.com and 555-0142", "org-1", nil) + if !strings.Contains(s, "[email:redacted]") || !strings.Contains(s, "[phone:redacted]") { + t.Errorf("no-key fallback did not redact: %q", s) + } + if strings.Contains(s, "a@b.com") || strings.Contains(s, "0142") { + t.Errorf("no-key fallback leaked the raw value: %q", s) + } +} + +func TestHashIsShortHex(t *testing.T) { + tok := tokenWithKey(PiiEmail, "a@b.com", "org-1", []byte(testKey)) + hexPart := strings.TrimSuffix(strings.TrimPrefix(tok, "[email:"), "]") + if len(hexPart) != hashHexLen { + t.Errorf("hash length %d, want %d: %q", len(hexPart), hashHexLen, tok) + } + for _, c := range hexPart { + if !strings.ContainsRune("0123456789abcdef", c) { + t.Errorf("non-hex char %q in %q", c, tok) + } + } +} + +func TestPiiTokenMatchesWhatScrubbingWrote(t *testing.T) { + // The searchability contract: hashing a typed query term must produce + // exactly the token stored in the span. + scrubbed := scrubT("mail a@b.com now", "org-7") + term := tokenWithKey(PiiEmail, "A@B.com ", "org-7", []byte(testKey)) + if !strings.Contains(scrubbed, term) { + t.Errorf("search seam broken: %q does not contain %q", scrubbed, term) + } +} + +func TestOrdinaryNumbersAreNotMistakenForPhones(t *testing.T) { + // The phone pattern requires a separator before the last 4 digits; ids, + // versions, dates and amounts must survive intact. + for _, s := range []string{ + "took 1234 ms", + "version 1.2.3", + "2026-08-15T12:00:00Z", + "total 1234.5678", + "trace 0af7651916cd43dd8448eb211c80319c", + } { + if got := scrubT(s, "org-1"); got != s { + t.Errorf("false positive on %q -> %q", s, got) + } + } +} + +func TestSetPiiHashKeyIsSetOnceAndRejectsEmpty(t *testing.T) { + if SetPiiHashKey(nil) { + t.Error("empty key should be rejected") + } + piiKeyMu.Lock() + saved := piiKey + piiKey = nil + piiKeyMu.Unlock() + defer func() { + piiKeyMu.Lock() + piiKey = saved + piiKeyMu.Unlock() + }() + + if !SetPiiHashKey([]byte("first-key")) { + t.Fatal("first install should succeed") + } + if SetPiiHashKey([]byte("second-key")) { + t.Error("rotation should be refused") + } + if string(piiHashKey()) != "first-key" { + t.Errorf("key was rotated: %q", piiHashKey()) + } +} + +func TestScrubHeadersForOrgSaltsValues(t *testing.T) { + in := map[string]string{"X-Note": "reply to a@b.com"} + a := ScrubHeadersForOrg(in, "org-1")["X-Note"] + b := ScrubHeadersForOrg(in, "org-2")["X-Note"] + if strings.Contains(a, "a@b.com") || strings.Contains(b, "a@b.com") { + t.Fatalf("raw email in header output: %q / %q", a, b) + } + // With no key installed both redact; with one installed they must differ. + if piiHashKey() != nil && a == b { + t.Errorf("header hashes not org-salted: %q == %q", a, b) + } +} + +// TestCrossSDKParityVectors pins the exact bytes every SDK must produce. +// Computed independently (python `hmac.new(key, org\0kind\0normalized, +// "sha256")`) and asserted verbatim in all five SDKs. If any SDK's message +// framing, normalization or truncation drifts, exactly one of these breaks. +func TestCrossSDKParityVectors(t *testing.T) { + cases := []struct { + kind PiiKind + raw string + org string + want string + }{ + {PiiEmail, "a@b.com", "org-1", "[email:02ea437f]"}, + {PiiEmail, "A@B.COM ", "org-1", "[email:02ea437f]"}, + {PiiEmail, "a@b.com", "org-2", "[email:fd96f7dc]"}, + {PiiEmail, "a@b.com", "", "[email:453b154f]"}, + {PiiPhone, "(415) 555-0142", "org-1", "[phone:415a9aea]"}, + {PiiPhone, "415-555-0142", "org-1", "[phone:415a9aea]"}, + {PiiAddress, "1600 Pennsylvania Ave", "org-1", "[address:c5351f4a]"}, + } + for _, c := range cases { + if got := tokenWithKey(c.kind, c.raw, c.org, []byte(testKey)); got != c.want { + t.Errorf("tokenWithKey(%s, %q, %q) = %s, want %s", c.kind, c.raw, c.org, got, c.want) + } + } +} diff --git a/packages/core/README.md b/packages/core/README.md index 25c6687..4fcab9c 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -141,3 +141,30 @@ wrapper covers Groq / DeepSeek / Azure / any OpenAI-compatible gateway via ## License MIT + +## PII scrubbing + +Two classes, handled differently: + +- **Credentials** (`Bearer …`, `password=`, `token`/`api_key`/`secret=`, `sk-…`) + are **dropped**. A hash of a live token is still a token oracle. +- **Personal identifiers** (email, phone, street address) are **hashed**: + `a@b.com` → `[email:9f2a41c8]`. The type prefix stays visible, so you can see + _what kind_ of value was there and that two spans carry the _same_ one — + without ever seeing it. + +The hash is **HMAC-SHA256**, not a bare digest (emails and phones are a small +enumerable space a rainbow table reverses in seconds), and the org id is mixed +into the message so the same value hashes **differently in different orgs**. + +Set the key with `SMOOAI_OBSERVABILITY_PII_HASH_KEY` (read by the bootstrap) or +`setPiiHashKey(...)` (the browser bundle has no env — call it explicitly). **With no key, personal identifiers are fully redacted** +(`[email:redacted]`) rather than hashed under a guessable one. + +⚠️ **The key and the org id are load-bearing.** Rotating either silently breaks +correlation with every hash already stored — treat the key as permanent, and do +not reuse a secret that rotates on a schedule. + +All five SDKs (TypeScript, Rust, Go, Python, .NET) emit byte-identical tokens +for the same key/org/value; the shared vectors are asserted in each SDK's PII +test suite. diff --git a/packages/core/src/__tests__/bootstrap.test.ts b/packages/core/src/__tests__/bootstrap.test.ts index 62e8f3d..98c96f2 100644 --- a/packages/core/src/__tests__/bootstrap.test.ts +++ b/packages/core/src/__tests__/bootstrap.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { _resetBootstrapForTests, bootstrapObservability } from '../bootstrap'; import { _resetOtelSdkForTests } from '../otel'; +import { _resetPiiHashKeyForTests, _tokenWithKey, piiToken, scrubString } from '../pii'; // Capture stderr writes so we can assert on the bootstrap warning paths // without polluting test output. @@ -143,3 +144,34 @@ describe('bootstrapObservability', () => { ).resolves.toBeDefined(); }); }); + +describe('bootstrapObservability — PII hash key', () => { + it('installs the key from SMOOAI_OBSERVABILITY_PII_HASH_KEY before anything can emit', async () => { + const previous = process.env.SMOOAI_OBSERVABILITY_PII_HASH_KEY; + process.env.SMOOAI_OBSERVABILITY_PII_HASH_KEY = 'env-supplied-key'; + _resetPiiHashKeyForTests(); + try { + // Disabled bootstrap still installs the key — the scrubber runs + // whether or not there is an exporter behind it. + await bootstrapObservability({ disabled: true }); + expect(piiToken('email', 'a@b.com', 'org-1')).toBe(_tokenWithKey('email', 'a@b.com', 'org-1', new TextEncoder().encode('env-supplied-key'))); + } finally { + _resetPiiHashKeyForTests(); + if (previous === undefined) delete process.env.SMOOAI_OBSERVABILITY_PII_HASH_KEY; + else process.env.SMOOAI_OBSERVABILITY_PII_HASH_KEY = previous; + } + }); + + it('leaves PII redacted when no key is configured', async () => { + const previous = process.env.SMOOAI_OBSERVABILITY_PII_HASH_KEY; + delete process.env.SMOOAI_OBSERVABILITY_PII_HASH_KEY; + _resetPiiHashKeyForTests(); + try { + await bootstrapObservability({ disabled: true }); + expect(scrubString('mail a@b.com')).toBe('mail [email:redacted]'); + } finally { + _resetPiiHashKeyForTests(); + if (previous !== undefined) process.env.SMOOAI_OBSERVABILITY_PII_HASH_KEY = previous; + } + }); +}); diff --git a/packages/core/src/__tests__/hmac-sha256.test.ts b/packages/core/src/__tests__/hmac-sha256.test.ts new file mode 100644 index 0000000..b189bb7 --- /dev/null +++ b/packages/core/src/__tests__/hmac-sha256.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest'; +import { hmacSha256, sha256, toHex } from '../hmac-sha256'; + +/** + * The hand-rolled sync primitive `pii.ts` depends on. These are the published + * FIPS 180-4 / RFC 4231 vectors — if this file is green the implementation is + * the real algorithm, not something that merely looks like a hash. + */ + +const enc = new TextEncoder(); +const hex = (b: Uint8Array) => toHex(b); + +describe('sha256', () => { + it('matches the FIPS 180-4 vectors', () => { + expect(hex(sha256(enc.encode('')))).toBe('e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'); + expect(hex(sha256(enc.encode('abc')))).toBe('ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad'); + expect(hex(sha256(enc.encode('abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq')))).toBe( + '248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1', + ); + }); + + it('handles the 55/56/64-byte padding boundaries', () => { + // 56 bytes is where the length field no longer fits in the same block — + // the classic off-by-one in a hand-rolled pad. + expect(hex(sha256(enc.encode('a'.repeat(55))))).toBe('9f4390f8d30c2dd92ec9f095b65e2b9ae9b0a925a5258e241c9f1e910f734318'); + expect(hex(sha256(enc.encode('a'.repeat(56))))).toBe('b35439a4ac6f0948b6d6f9e3c6af0f5f590ce20f1bde7090ef7970686ec6738a'); + expect(hex(sha256(enc.encode('a'.repeat(64))))).toBe('ffe054fe7ae0cb6dc65c3af9b61d5209f439851db43d0ba5997337df154668eb'); + }); + + it('hashes a million a-s (FIPS long vector)', () => { + expect(hex(sha256(enc.encode('a'.repeat(1_000_000))))).toBe('cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0'); + }); +}); + +describe('hmacSha256', () => { + it('matches RFC 4231 test case 1', () => { + const key = new Uint8Array(20).fill(0x0b); + expect(hex(hmacSha256(key, enc.encode('Hi There')))).toBe('b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7'); + }); + + it('matches RFC 4231 test case 2', () => { + expect(hex(hmacSha256(enc.encode('Jefe'), enc.encode('what do ya want for nothing?')))).toBe( + '5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843', + ); + }); + + it('matches RFC 4231 test case 3', () => { + const key = new Uint8Array(20).fill(0xaa); + const data = new Uint8Array(50).fill(0xdd); + expect(hex(hmacSha256(key, data))).toBe('773ea91e36800e46854db8ebd09181a72959098b3ef8c122d9635514ced565fe'); + }); + + it('matches RFC 4231 test case 6 — key longer than one block', () => { + const key = new Uint8Array(131).fill(0xaa); + expect(hex(hmacSha256(key, enc.encode('Test Using Larger Than Block-Size Key - Hash Key First')))).toBe( + '60e431591ee0b67f0d8a26aacbf5b77f8e0bc6213728c5140546040f0ee37f54', + ); + }); +}); + +describe('toHex', () => { + it('truncates to the requested length', () => { + const bytes = new Uint8Array([0x00, 0xab, 0xff, 0x10]); + expect(toHex(bytes)).toBe('00abff10'); + expect(toHex(bytes, 4)).toBe('00ab'); + // Leading zeroes must survive — a naive toString(16) drops them. + expect(toHex(new Uint8Array([0x01, 0x02]))).toBe('0102'); + }); +}); diff --git a/packages/core/src/__tests__/pii.test.ts b/packages/core/src/__tests__/pii.test.ts index d92cb90..2da3a27 100644 --- a/packages/core/src/__tests__/pii.test.ts +++ b/packages/core/src/__tests__/pii.test.ts @@ -1,18 +1,179 @@ -import { describe, expect, it } from 'vitest'; -import { scrubHeaders, scrubString } from '../pii'; +import { beforeEach, describe, expect, it } from 'vitest'; +import { _resetPiiHashKeyForTests, _scrubWithKey, _tokenWithKey, piiToken, scrubHeaders, scrubHeadersForOrg, scrubString, setPiiHashKey } from '../pii'; -describe('scrubString', () => { +/** + * Mirrors `rust/observability/src/pii.rs`'s test suite so parity across the + * five SDKs is auditable case-by-case. + * + * The hashing tests drive `_scrubWithKey` / `_tokenWithKey` rather than + * installing the process-wide key: `setPiiHashKey` is set-once and vitest runs + * the file in one module instance, so a global write would make the suite + * order-dependent. + */ + +const enc = new TextEncoder(); +const KEY = enc.encode('test-hmac-key-not-a-real-secret'); +const OTHER_KEY = enc.encode('a-different-test-hmac-key'); + +const scrub = (input: string, org = 'org-1') => _scrubWithKey(input, org, KEY); + +// ---- credentials: dropped, never hashed ---------------------------------- + +describe('scrubString — credentials', () => { it('redacts Bearer tokens', () => { expect(scrubString('Authorization: Bearer abc.def.ghi')).toBe('Authorization: Bearer [redacted]'); }); it('redacts password=', () => { expect(scrubString('?password=hunter2&x=1')).toBe('?password=[redacted]&x=1'); }); + it('redacts token= while keeping the key', () => { + const out = scrubString('api_key=supersecretvalue'); + expect(out).toBe('api_key=[redacted]'); + expect(out).not.toContain('supersecretvalue'); + }); it('redacts sk-... API keys', () => { expect(scrubString('key=sk-AAAAAAAAAAAAAAAAAAAAAAAAAAAA')).toContain('sk-[redacted]'); }); + + it('drops credentials rather than hashing them', () => { + // A live token must never become a correlatable handle — hashing one + // still yields an oracle you can test candidate tokens against. + const out = scrub('Bearer abc.def-ghi_123 password=hunter2 sk-ABCDEFGHIJKLMNOPQRSTUVWX'); + expect(out).toContain('Bearer [redacted]'); + expect(out).toContain('password=[redacted]'); + expect(out).toContain('sk-[redacted]'); + for (const shape of ['[token:', '[credential:', '[email:', '[phone:']) { + expect(out).not.toContain(shape); + } + // And a personal identifier hiding inside a secret goes with it. + const inSecret = scrub('token=a@b.com'); + expect(inSecret).not.toContain('a@b.com'); + expect(inSecret).not.toContain('[email:'); + }); +}); + +// ---- personal identifiers: hashed, prefix preserved ---------------------- + +describe('scrubString — personal identifiers', () => { + it('hashes emails keeping the type prefix in place', () => { + const out = scrub('contact me at Alice@Example.com please'); + expect(out.toLowerCase()).not.toContain('alice@example.com'); + expect(out.startsWith('contact me at [email:')).toBe(true); + expect(out.endsWith('] please')).toBe(true); + }); + + it.each(['555-0142', '(415) 555-0142', '+1 415-555-0142'])('hashes phone number %s', (raw) => { + const out = scrub(`call ${raw} today`); + expect(out).toContain('[phone:'); + expect(out).not.toContain('0142'); + }); + + it('hashes street addresses', () => { + const out = scrub('ship to 1600 Pennsylvania Ave, Washington'); + expect(out).toContain('[address:'); + expect(out).not.toContain('Pennsylvania'); + }); + + it('is stable for the same value in the same org', () => { + expect(scrub('a@b.com')).toBe(scrub('a@b.com')); + // …and correlation survives formatting differences in phones. + expect(scrub('(415) 555-0142')).toBe(scrub('415-555-0142')); + }); + + it('hashes the same value differently in a different org', () => { + const a = scrub('a@b.com', 'org-1'); + const b = scrub('a@b.com', 'org-2'); + expect(a).not.toBe(b); + expect(a.startsWith('[email:')).toBe(true); + expect(b.startsWith('[email:')).toBe(true); + }); + + it('hashes differently under a different key', () => { + expect(_scrubWithKey('a@b.com', 'org-1', KEY)).not.toBe(_scrubWithKey('a@b.com', 'org-1', OTHER_KEY)); + }); + + it('redacts rather than hashing when no key is installed', () => { + // Fail safe: an unkeyed digest of an email is rainbow-tabled instantly. + const out = _scrubWithKey('a@b.com and 555-0142', 'org-1', null); + expect(out).toContain('[email:redacted]'); + expect(out).toContain('[phone:redacted]'); + expect(out).not.toContain('a@b.com'); + expect(out).not.toContain('0142'); + }); + + it('emits a short lowercase hex hash', () => { + const token = _tokenWithKey('email', 'a@b.com', 'org-1', KEY); + const hex = token.slice('[email:'.length, -1); + expect(hex).toHaveLength(8); + expect(hex).toMatch(/^[0-9a-f]{8}$/); + }); + + it('leaves ordinary numbers alone', () => { + // The phone pattern requires a separator before the last 4 digits; ids, + // versions, dates and amounts must survive intact. + for (const value of ['took 1234 ms', 'version 1.2.3', '2026-08-15T12:00:00Z', 'total 1234.5678', 'trace 0af7651916cd43dd8448eb211c80319c']) { + expect(scrub(value)).toBe(value); + } + }); +}); + +// ---- the search seam ------------------------------------------------------ + +describe('piiToken', () => { + beforeEach(() => { + _resetPiiHashKeyForTests(); + }); + + it('produces exactly what scrubbing wrote', () => { + // The searchability contract: hashing a typed query term must produce + // exactly the token stored in the span. + const scrubbed = _scrubWithKey('mail a@b.com now', 'org-7', KEY); + expect(scrubbed).toContain(_tokenWithKey('email', 'A@B.com ', 'org-7', KEY)); + }); + + it('redacts when no key is installed', () => { + expect(piiToken('email', 'a@b.com', 'org-1')).toBe('[email:redacted]'); + }); +}); + +// ---- cross-SDK byte parity ----------------------------------------------- + +describe('cross-SDK parity vectors', () => { + // Computed independently (python `hmac.new(key, org\0kind\0normalized, + // "sha256")`) and asserted verbatim in all five SDKs. If any SDK's message + // framing, normalization or truncation drifts, exactly one of these breaks. + it.each([ + ['email', 'a@b.com', 'org-1', '[email:02ea437f]'], + ['email', 'A@B.COM ', 'org-1', '[email:02ea437f]'], + ['email', 'a@b.com', 'org-2', '[email:fd96f7dc]'], + ['email', 'a@b.com', '', '[email:453b154f]'], + ['phone', '(415) 555-0142', 'org-1', '[phone:415a9aea]'], + ['phone', '415-555-0142', 'org-1', '[phone:415a9aea]'], + ['address', '1600 Pennsylvania Ave', 'org-1', '[address:c5351f4a]'], + ] as const)('%s %s in %s', (kind, raw, org, expected) => { + expect(_tokenWithKey(kind, raw, org, KEY)).toBe(expected); + }); +}); + +// ---- set-once key --------------------------------------------------------- + +describe('setPiiHashKey', () => { + beforeEach(() => { + _resetPiiHashKeyForTests(); + }); + + it('rejects an empty key and refuses rotation', () => { + expect(setPiiHashKey('')).toBe(false); + expect(setPiiHashKey('first-key')).toBe(true); + expect(setPiiHashKey('second-key')).toBe(false); + // The first key is what is still in force. + expect(piiToken('email', 'a@b.com', 'o')).toBe(_tokenWithKey('email', 'a@b.com', 'o', enc.encode('first-key'))); + _resetPiiHashKeyForTests(); + }); }); +// ---- headers -------------------------------------------------------------- + describe('scrubHeaders', () => { it('redacts known sensitive headers', () => { const out = scrubHeaders({ authorization: 'Bearer abc', 'x-api-key': '12345', accept: 'application/json' })!; @@ -20,4 +181,10 @@ describe('scrubHeaders', () => { expect(out['x-api-key']).toBe('[redacted]'); expect(out.accept).toBe('application/json'); }); + + it('never leaks a raw email through a non-sensitive header', () => { + const out = scrubHeadersForOrg({ 'X-Note': 'reply to a@b.com' }, 'org-1')!; + expect(out['X-Note']).not.toContain('a@b.com'); + expect(out['X-Note']).toContain('[email:'); + }); }); diff --git a/packages/core/src/bootstrap/index.ts b/packages/core/src/bootstrap/index.ts index 9844ae2..8c9f82f 100644 --- a/packages/core/src/bootstrap/index.ts +++ b/packages/core/src/bootstrap/index.ts @@ -58,6 +58,13 @@ * SMOOAI_OBSERVABILITY_DISABLED — set to "1"/"true" to skip * bootstrap entirely (useful in * tests). + * SMOOAI_OBSERVABILITY_PII_HASH_KEY — HMAC key used to hash emails / + * phones / addresses in scrubbed + * strings (see `../pii`). Unset + * means personal identifiers are + * fully redacted rather than hashed. + * Set once; rotating it forks every + * stored correlation. * * ## Behavior * @@ -74,6 +81,7 @@ import { TokenProvider } from '../auth/token-provider'; import { Client } from '../node'; import { setupOtelSdk, type OtelSdkHandle, type SetupOtelOptions } from '../otel'; +import { setPiiHashKey } from '../pii'; const TOKEN_REFRESH_INTERVAL_MS = 55 * 60 * 1000; // < openauth's 1h JWT TTL @@ -134,8 +142,14 @@ export async function bootstrapObservability(overrides: Partial = environment: overrides.environment ?? process.env.SMOOAI_OBSERVABILITY_ENVIRONMENT ?? process.env.STAGE ?? process.env.NODE_ENV, release: overrides.release ?? process.env.SMOOAI_OBSERVABILITY_RELEASE ?? process.env.GIT_SHA ?? process.env.LAMBDA_FUNCTION_VERSION ?? 'dev', disabled: overrides.disabled ?? truthy(process.env.SMOOAI_OBSERVABILITY_DISABLED), + piiHashKey: overrides.piiHashKey ?? process.env.SMOOAI_OBSERVABILITY_PII_HASH_KEY, }; + // Before anything can emit: a scrubbed string written without this key + // redacts PII outright, so installing it late would silently produce a + // window of uncorrelatable spans rather than an error. + if (env.piiHashKey) setPiiHashKey(env.piiHashKey); + if (env.disabled) { bootstrapped = { installed: false, exporting: false, otel: null, stopRefresh: () => {} }; return bootstrapped; @@ -263,6 +277,12 @@ export interface BootstrapEnv { environment?: string; release?: string; disabled?: boolean; + /** + * HMAC key used to hash emails / phones / addresses in scrubbed strings + * (see `../pii`). Unset means personal identifiers are fully redacted + * rather than hashed. + */ + piiHashKey?: string; } interface RefreshConfig { diff --git a/packages/core/src/gen-ai-attributes.ts b/packages/core/src/gen-ai-attributes.ts index 3a8feaa..d28182f 100644 --- a/packages/core/src/gen-ai-attributes.ts +++ b/packages/core/src/gen-ai-attributes.ts @@ -108,11 +108,14 @@ export function setGenAIAttributes(span: Span, attrs: GenAIAttributes): void { * prompts and tool arguments are the single most PII-dense payload this SDK can * touch, so raw content never reaches the wire. * - * ponytail: `scrubString` in TS is credentials-only today (Bearer tokens, api - * keys, `password=`). Keyed per-org hashing of names / emails / phones exists in - * Rust (`rust/observability/src/pii.rs`) and is being ported to TS in a parallel - * PR — when it lands, this call site inherits it for free because it already - * routes through the SDK's one scrub entry point. Do not scrub inline here. + * That covers both classes: credentials (Bearer tokens, api keys, `password=`) + * are dropped, and emails / phones / addresses are hashed per-org. This call + * site inherited the hashing for free by routing through the SDK's one scrub + * entry point. Do not scrub inline here. + * + * ponytail: uses the org-less `scrubString`, so hashes are salted with the + * empty org — there is no org id in hand at this call site. Switch to + * `scrubStringForOrg` if one ever reaches here. */ export function recordGenAIMessage( span: Span, diff --git a/packages/core/src/hmac-sha256.ts b/packages/core/src/hmac-sha256.ts new file mode 100644 index 0000000..eb542fe --- /dev/null +++ b/packages/core/src/hmac-sha256.ts @@ -0,0 +1,130 @@ +/** + * Minimal synchronous SHA-256 + HMAC-SHA256. + * + * Why hand-rolled rather than `node:crypto` or WebCrypto: `scrubString` is + * synchronous and runs in **both** the browser and Node bundles. `node:crypto` + * breaks the browser build, and WebCrypto's `subtle.sign` is async-only — it + * cannot be called from a sync scrubber. This is the only sync primitive that + * works in both runtimes without adding a dependency to a published SDK. + * + * Correctness is pinned by the RFC 4231 / FIPS 180-4 vectors in + * `__tests__/hmac-sha256.test.ts`. Do not "optimize" this file without + * re-running them. + * + * Not constant-time, and not intended for verifying MACs against attacker- + * supplied values — the only consumer is `pii.ts`, which hashes values it + * already holds. + */ + +const K = /* @__PURE__ */ new Uint32Array([ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, + 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, + 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, + 0x92722c85, 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +]); + +const BLOCK_BYTES = 64; + +function rotr(x: number, n: number): number { + return (x >>> n) | (x << (32 - n)); +} + +/** FIPS 180-4 SHA-256. Returns the 32-byte digest. */ +export function sha256(message: Uint8Array): Uint8Array { + const h = new Uint32Array([0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19]); + + // Pad to a multiple of 64 bytes: 0x80, zeroes, then the 64-bit big-endian + // bit length. + const paddedLength = Math.ceil((message.length + 9) / BLOCK_BYTES) * BLOCK_BYTES; + const padded = new Uint8Array(paddedLength); + padded.set(message); + padded[message.length] = 0x80; + const view = new DataView(padded.buffer); + const bitLength = message.length * 8; + view.setUint32(paddedLength - 8, Math.floor(bitLength / 0x100000000), false); + view.setUint32(paddedLength - 4, bitLength >>> 0, false); + + const w = new Uint32Array(64); + for (let offset = 0; offset < paddedLength; offset += BLOCK_BYTES) { + for (let i = 0; i < 16; i++) w[i] = view.getUint32(offset + i * 4, false); + for (let i = 16; i < 64; i++) { + const x = w[i - 15]!; + const y = w[i - 2]!; + const s0 = rotr(x, 7) ^ rotr(x, 18) ^ (x >>> 3); + const s1 = rotr(y, 17) ^ rotr(y, 19) ^ (y >>> 10); + w[i] = (w[i - 16]! + s0 + w[i - 7]! + s1) >>> 0; + } + + let a = h[0]!; + let b = h[1]!; + let c = h[2]!; + let d = h[3]!; + let e = h[4]!; + let f = h[5]!; + let g = h[6]!; + let hh = h[7]!; + + for (let i = 0; i < 64; i++) { + const s1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25); + const ch = (e & f) ^ (~e & g); + const temp1 = (hh + s1 + ch + K[i]! + w[i]!) >>> 0; + const s0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22); + const maj = (a & b) ^ (a & c) ^ (b & c); + const temp2 = (s0 + maj) >>> 0; + + hh = g; + g = f; + f = e; + e = (d + temp1) >>> 0; + d = c; + c = b; + b = a; + a = (temp1 + temp2) >>> 0; + } + + h[0] = (h[0]! + a) >>> 0; + h[1] = (h[1]! + b) >>> 0; + h[2] = (h[2]! + c) >>> 0; + h[3] = (h[3]! + d) >>> 0; + h[4] = (h[4]! + e) >>> 0; + h[5] = (h[5]! + f) >>> 0; + h[6] = (h[6]! + g) >>> 0; + h[7] = (h[7]! + hh) >>> 0; + } + + const digest = new Uint8Array(32); + const digestView = new DataView(digest.buffer); + for (let i = 0; i < 8; i++) digestView.setUint32(i * 4, h[i]!, false); + return digest; +} + +function concat(a: Uint8Array, b: Uint8Array): Uint8Array { + const out = new Uint8Array(a.length + b.length); + out.set(a); + out.set(b, a.length); + return out; +} + +/** RFC 2104 HMAC-SHA256. Returns the 32-byte MAC. */ +export function hmacSha256(key: Uint8Array, message: Uint8Array): Uint8Array { + const block = new Uint8Array(BLOCK_BYTES); + block.set(key.length > BLOCK_BYTES ? sha256(key) : key); + + const inner = new Uint8Array(BLOCK_BYTES); + const outer = new Uint8Array(BLOCK_BYTES); + for (let i = 0; i < BLOCK_BYTES; i++) { + inner[i] = block[i]! ^ 0x36; + outer[i] = block[i]! ^ 0x5c; + } + return sha256(concat(outer, sha256(concat(inner, message)))); +} + +/** Lowercase hex of `bytes`, truncated to `length` characters. */ +export function toHex(bytes: Uint8Array, length = bytes.length * 2): string { + let out = ''; + for (let i = 0; i < bytes.length && out.length < length; i++) { + out += bytes[i]!.toString(16).padStart(2, '0'); + } + return out.slice(0, length); +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 7ac0175..31cc163 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -38,3 +38,8 @@ export { type TelemetrySettingsProvider, } from './telemetry-settings'; export { parseTraceparent, formatTraceparent, type TraceContext } from './traceparent'; +// PII scrubbing / hashing. `setPiiHashKey` MUST be called before anything is +// captured — without a key, emails / phones / addresses are redacted outright +// rather than hashed. On Node `bootstrapObservability` does it from +// SMOOAI_OBSERVABILITY_PII_HASH_KEY; the browser has no env, so call it here. +export { scrubString, scrubStringForOrg, scrubHeaders, scrubHeadersForOrg, setPiiHashKey, piiToken, type PiiKind } from './pii'; diff --git a/packages/core/src/pii.ts b/packages/core/src/pii.ts index 2393f6e..236ebb0 100644 --- a/packages/core/src/pii.ts +++ b/packages/core/src/pii.ts @@ -1,31 +1,192 @@ /** * PII scrubbing — applied to message strings, breadcrumb messages, and headers - * before transport. Stays opinionated and minimal; tenants can extend in - * `beforeSend`. + * before transport. Mirrors `rust/observability/src/pii.rs`; the semantics are + * identical across the five SDKs. + * + * Two classes, handled differently on purpose: + * + * - **Credentials** (`Bearer …`, `password=`, `token`/`api_key`/`secret=`, + * `sk-…`) are **dropped**. A hash of a live token is still a token oracle, + * and there is no correlation value in a secret. + * - **Personal identifiers** (email, phone, street address) are **hashed**, not + * dropped: `a@b.com` → `[email:9f2a41c8]`. That keeps the one question worth + * asking — "are these two spans the same person?" — answerable while storing + * nothing reversible. + * + * The hash is **HMAC-SHA256, keyed**, not a bare digest: emails and phone + * numbers are a small enumerable space that a rainbow table reverses in + * seconds. The org id is mixed into the HMAC message, so identical PII hashes + * **differently in different orgs**. + * + * **The key and the org salt are load-bearing and must not rotate casually.** + * Rotating either silently breaks correlation with every previously stored + * hash. Supply the key once at startup via `SMOOAI_OBSERVABILITY_PII_HASH_KEY` + * (read by `bootstrapObservability`) or {@link setPiiHashKey} — the browser + * bundle has no env, so it must call `setPiiHashKey` explicitly. **With no key + * configured, personal identifiers are fully redacted** (`[email:redacted]`) + * rather than hashed under a guessable one — fail safe, never fail open. + * + * Pattern matching never catches everything. It fails safe, which is the right + * default for data we persist from end-user conversations; tenants can extend + * in `beforeSend`. */ -const PII_PATTERNS: Array<{ re: RegExp; replacement: string }> = [ +import { hmacSha256, toHex } from './hmac-sha256'; + +/** + * Credentials. Matched FIRST so a personal identifier sitting inside a secret + * (`token=a@b.com`) is dropped with the secret rather than surviving as a hash. + */ +const CREDENTIAL_PATTERNS: Array<{ re: RegExp; replacement: string | ((match: string) => string) }> = [ { re: /Bearer\s+[A-Za-z0-9._-]+/gi, replacement: 'Bearer [redacted]' }, { re: /\b(?:password|passwd|pwd)["']?\s*[:=]\s*["']?[^"'&\s]+/gi, replacement: 'password=[redacted]' }, - { re: /\b(?:token|api[-_]?key|apikey|secret)["']?\s*[:=]\s*["']?[^"'&\s]+/gi, replacement: '$&'.replace(/=.*/, '=[redacted]') }, + // Key-preserving: keep everything up to and including the `=`/`:` + // separator, redact the rest. + { + re: /\b(?:token|api[-_]?key|apikey|secret)["']?\s*[:=]\s*["']?[^"'&\s]+/gi, + replacement: (match: string) => match.replace(/^(.*?[:=]).*$/, '$1[redacted]'), + }, { re: /\bsk-[A-Za-z0-9]{20,}/g, replacement: 'sk-[redacted]' }, ]; +/** + * The class of personal identifier a match represents. Drives both the visible + * prefix in the output token and the normalization applied before hashing, so + * `(415) 555-0142` and `415-555-0142` correlate. + */ +export type PiiKind = 'email' | 'phone' | 'address'; + +function normalize(kind: PiiKind, raw: string): string { + switch (kind) { + case 'email': + return raw.trim().toLowerCase(); + case 'phone': + // Digits only: formatting must not fork the hash. + return raw.replace(/[^0-9]/g, ''); + case 'address': + // Case-fold and collapse runs of whitespace. + return raw.trim().replace(/\s+/g, ' ').toLowerCase(); + } +} + +/** + * Personal identifiers — hashed, not dropped. Order matters only in that these + * all run after {@link CREDENTIAL_PATTERNS}. + */ +const PERSONAL_PATTERNS: Array<{ kind: PiiKind; re: RegExp }> = [ + { kind: 'email', re: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+\b/gi }, + // Phone: optional country code / area code, then the 3-4 local pair. A + // separator is REQUIRED so bare digit runs (ids, timestamps, amounts) don't + // get eaten. + { kind: 'phone', re: /(?:\+\d{1,3}[ .-]?)?(?:\(\d{3}\)[ .-]?|\b\d{3}[ .-])?\b\d{3}[ .-]\d{4}\b/g }, + // US-style street address: house number, 1-3 words, a street suffix. + { + kind: 'address', + re: /\b\d{1,6}\s+(?:[A-Za-z0-9.'-]+\s+){0,3}(?:street|st|avenue|ave|road|rd|boulevard|blvd|lane|ln|drive|dr|court|ct|way|terrace|ter|place|pl|circle|cir|highway|hwy|parkway|pkwy|square|sq)\b\.?/gi, + }, +]; + const SENSITIVE_HEADERS = new Set(['authorization', 'cookie', 'set-cookie', 'x-api-key', 'x-auth-token']); +/** + * Hex characters kept from the HMAC. Long enough that collisions are rare + * across an org's traces, short enough to read in a span attribute. + */ +const HASH_HEX_LEN = 8; + +const encoder = new TextEncoder(); + +let piiHashKey: Uint8Array | null = null; + +/** + * Install the process-wide HMAC key used to hash personal identifiers. + * Idempotent and set-once: returns `false` if a key was already installed (the + * existing key is kept — a mid-process rotation would silently fork every + * correlation) or if `key` is empty. `bootstrapObservability` calls this from + * `SMOOAI_OBSERVABILITY_PII_HASH_KEY`. + */ +export function setPiiHashKey(key: string | Uint8Array): boolean { + const bytes = typeof key === 'string' ? encoder.encode(key) : key; + if (bytes.length === 0 || piiHashKey !== null) return false; + piiHashKey = bytes; + return true; +} + +/** Test seam — clears the set-once key so set-once itself can be asserted. */ +export function _resetPiiHashKeyForTests(): void { + piiHashKey = null; +} + +/** + * Hash one known-personal value into its scrubbed token — the same token + * {@link scrubStringForOrg} would have written. This is how a UI search box + * finds stored hashes: hash the typed term with the same org and match. + * + * Returns `[:redacted]` when no key is installed. + */ +export function piiToken(kind: PiiKind, raw: string, orgId: string): string { + return tokenWithKey(kind, raw, orgId, piiHashKey); +} + +function tokenWithKey(kind: PiiKind, raw: string, orgId: string, key: Uint8Array | null): string { + if (!key || key.length === 0) return `[${kind}:redacted]`; + // orgId in the HMAC message IS the per-org salt. The kind is in there too so + // a phone and an address that normalize alike can't collide. + // NUL separators (written as explicit escapes — a literal NUL in source is + // invisible and formatter-fragile) so no field can impersonate a boundary. + const message = encoder.encode(`${orgId}\u0000${kind}\u0000${normalize(kind, raw)}`); + return `[${kind}:${toHex(hmacSha256(key, message), HASH_HEX_LEN)}]`; +} + +/** + * Scrub a free-form string with no org context — credentials dropped, personal + * identifiers hashed under the empty org salt. Prefer {@link scrubStringForOrg} + * wherever an org id is in hand, so hashes can't be correlated across tenants. + */ export function scrubString(input: string): string { + return scrubWithKey(input, '', piiHashKey); +} + +/** Scrub a free-form string, salting personal-identifier hashes with `orgId`. */ +export function scrubStringForOrg(input: string, orgId: string): string { + return scrubWithKey(input, orgId, piiHashKey); +} + +/** @internal Test seam — scrub with an explicit key instead of the global one. */ +export function _scrubWithKey(input: string, orgId: string, key: Uint8Array | null): string { + return scrubWithKey(input, orgId, key); +} + +/** @internal Test seam — token with an explicit key instead of the global one. */ +export function _tokenWithKey(kind: PiiKind, raw: string, orgId: string, key: Uint8Array | null): string { + return tokenWithKey(kind, raw, orgId, key); +} + +function scrubWithKey(input: string, orgId: string, key: Uint8Array | null): string { let out = input; - for (const { re, replacement } of PII_PATTERNS) { - out = out.replace(re, replacement); + for (const { re, replacement } of CREDENTIAL_PATTERNS) { + out = typeof replacement === 'string' ? out.replace(re, replacement) : out.replace(re, (match: string) => replacement(match)); + } + for (const { kind, re } of PERSONAL_PATTERNS) { + out = out.replace(re, (match) => tokenWithKey(kind, match, orgId, key)); } return out; } +/** + * Scrub a header map: sensitive header names are fully redacted, all other + * values are run through {@link scrubString}. + */ export function scrubHeaders(headers: Record | undefined): Record | undefined { + return scrubHeadersForOrg(headers, ''); +} + +/** {@link scrubHeaders} with an org salt for the personal-identifier hashes. */ +export function scrubHeadersForOrg(headers: Record | undefined, orgId: string): Record | undefined { if (!headers) return headers; const out: Record = {}; for (const [k, v] of Object.entries(headers)) { - out[k] = SENSITIVE_HEADERS.has(k.toLowerCase()) ? '[redacted]' : scrubString(v); + out[k] = SENSITIVE_HEADERS.has(k.toLowerCase()) ? '[redacted]' : scrubStringForOrg(v, orgId); } return out; } diff --git a/python/README.md b/python/README.md index d5425a3..e5fbaf5 100644 --- a/python/README.md +++ b/python/README.md @@ -118,6 +118,7 @@ Same names as the TS bootstrap: | `SMOOAI_OBSERVABILITY_SERVICE_NAME` | OTel `service.name` (default `smoo-service`) | | `SMOOAI_OBSERVABILITY_ENVIRONMENT` / `_RELEASE` | Deployment env / release id | | `SMOOAI_OBSERVABILITY_DISABLED` | `1`/`true` to skip bootstrap | +| `SMOOAI_OBSERVABILITY_PII_HASH_KEY` | HMAC key for hashing emails / phones / addresses (unset = redacted) | ## Development @@ -126,3 +127,30 @@ uv sync --all-extras --dev uv run ruff check . && uv run ruff format --check . uv run pytest ``` + +## PII scrubbing + +Two classes, handled differently: + +- **Credentials** (`Bearer …`, `password=`, `token`/`api_key`/`secret=`, `sk-…`) + are **dropped**. A hash of a live token is still a token oracle. +- **Personal identifiers** (email, phone, street address) are **hashed**: + `a@b.com` → `[email:9f2a41c8]`. The type prefix stays visible, so you can see + *what kind* of value was there and that two spans carry the *same* one — + without ever seeing it. + +The hash is **HMAC-SHA256**, not a bare digest (emails and phones are a small +enumerable space a rainbow table reverses in seconds), and the org id is mixed +into the message so the same value hashes **differently in different orgs**. + +Set the key with `SMOOAI_OBSERVABILITY_PII_HASH_KEY` (read by the bootstrap) or +`pii.set_pii_hash_key(...)`. **With no key, personal identifiers are fully redacted** +(`[email:redacted]`) rather than hashed under a guessable one. + +⚠️ **The key and the org id are load-bearing.** Rotating either silently breaks +correlation with every hash already stored — treat the key as permanent, and do +not reuse a secret that rotates on a schedule. + +All five SDKs (TypeScript, Rust, Go, Python, .NET) emit byte-identical tokens +for the same key/org/value; the shared vectors are asserted in each SDK's PII +test suite. diff --git a/python/src/smooai_observability/bootstrap/__init__.py b/python/src/smooai_observability/bootstrap/__init__.py index 45a2f98..e8a3dea 100644 --- a/python/src/smooai_observability/bootstrap/__init__.py +++ b/python/src/smooai_observability/bootstrap/__init__.py @@ -26,6 +26,10 @@ SMOOAI_OBSERVABILITY_RELEASE — default GIT_SHA / "dev". SMOOAI_OBSERVABILITY_DSN — webhook DSN for the Errors dashboard. SMOOAI_OBSERVABILITY_DISABLED — "1"/"true" to skip bootstrap entirely. +SMOOAI_OBSERVABILITY_PII_HASH_KEY — HMAC key used to hash emails / phones / + addresses in scrubbed strings (see + ``smooai_observability.pii``). Unset means + personal identifiers are fully redacted. """ from __future__ import annotations @@ -36,6 +40,7 @@ from ..auth.token_provider import TokenProvider, TokenProviderError from ..client import Client, ClientOptions +from ..pii import set_pii_hash_key from ..transport import Transport @@ -54,6 +59,7 @@ class BootstrapEnv: environment: str | None = None release: str | None = None disabled: bool = False + pii_hash_key: str | None = None @dataclass @@ -129,8 +135,15 @@ def bootstrap_observability( or os.environ.get("NODE_ENV"), release=o.release or os.environ.get("SMOOAI_OBSERVABILITY_RELEASE") or os.environ.get("GIT_SHA") or "dev", disabled=o.disabled or _truthy(os.environ.get("SMOOAI_OBSERVABILITY_DISABLED")), + pii_hash_key=o.pii_hash_key or os.environ.get("SMOOAI_OBSERVABILITY_PII_HASH_KEY"), ) + # Before anything can emit: a scrubbed string written without this key + # redacts PII outright, so installing it late would silently produce a + # window of uncorrelatable spans rather than an error. + if env.pii_hash_key: + set_pii_hash_key(env.pii_hash_key) + if env.disabled: _bootstrapped = BootstrapResult(installed=False) return _bootstrapped diff --git a/python/src/smooai_observability/pii.py b/python/src/smooai_observability/pii.py index 1ec5298..c90520a 100644 --- a/python/src/smooai_observability/pii.py +++ b/python/src/smooai_observability/pii.py @@ -1,29 +1,54 @@ """PII scrubbing — applied to message strings, breadcrumb messages, and headers -before transport. Stays opinionated and minimal; tenants can extend in -``before_send``. +before transport. Port of ``rust/observability/src/pii.rs``; the semantics are +identical across the five SDKs. + +Two classes, handled differently on purpose: + +- **Credentials** (``Bearer …``, ``password=``, ``token``/``api_key``/``secret=``, + ``sk-…``) are **dropped**. A hash of a live token is still a token oracle, and + there is no correlation value in a secret. +- **Personal identifiers** (email, phone, street address) are **hashed**, not + dropped: ``a@b.com`` → ``[email:9f2a41c8]``. That keeps the one question worth + asking — "are these two spans the same person?" — answerable while storing + nothing reversible. + +The hash is **HMAC-SHA256, keyed**, not a bare digest: emails and phone numbers +are a small enumerable space that a rainbow table reverses in seconds. The org +id is mixed into the HMAC message, so identical PII hashes **differently in +different orgs**. + +**The key and the org salt are load-bearing and must not rotate casually.** +Rotating either silently breaks correlation with every previously stored hash. +Supply the key once at startup via ``SMOOAI_OBSERVABILITY_PII_HASH_KEY`` (read +by :mod:`smooai_observability.bootstrap`) or :func:`set_pii_hash_key`. **With no +key configured, personal identifiers are fully redacted** (``[email:redacted]``) +rather than hashed under a guessable one — fail safe, never fail open. -Direct port of ``packages/core/src/pii.ts``. The regex patterns mirror the TS -source. One deliberate deviation: the TS ``token|api_key|secret`` pattern ships -a latent no-op replacement (``'$&'.replace(/=.*/, ...)`` evaluates to the -literal ``$&`` at module load, so the match is replaced with itself). A PII -scrubber that doesn't redact secrets is worse than useless, so the Python port -actually redacts the value while keeping the key — the clearly-intended -behavior. See the inline note on ``_TOKEN_RE``. +Pattern matching never catches everything. It fails safe, which is the right +default for data we persist from end-user conversations; tenants can extend in +``before_send``. """ from __future__ import annotations +import hmac import re +import threading +from enum import StrEnum + +# --- credentials: matched FIRST, dropped entirely -------------------------- +# +# A personal identifier sitting inside a secret (``token=a@b.com``) is dropped +# with the secret rather than surviving as a hash. -# Mirrors the TS PII_PATTERNS. Python's `re` uses `(?i)` / re.IGNORECASE for -# the `i` flag and re.sub iterates globally by default (no `g` flag needed). _BEARER_RE = re.compile(r"Bearer\s+[A-Za-z0-9._-]+", re.IGNORECASE) _PASSWORD_RE = re.compile( r"""\b(?:password|passwd|pwd)["']?\s*[:=]\s*["']?[^"'&\s]+""", re.IGNORECASE, ) -# TS source: the replacement string resolves to a no-op (`$&`). Python keeps -# the leading `key=`/`key:` and redacts only the value — the intended effect. +# The TS source's replacement string resolves to a no-op (`$&`). Python keeps +# the leading `key=`/`key:` and redacts only the value — the intended effect, +# and what the Rust reference does. _TOKEN_RE = re.compile( r"""\b(?P(?:token|api[-_]?key|apikey|secret)["']?\s*[:=]\s*["']?)[^"'&\s]+""", re.IGNORECASE, @@ -32,13 +57,138 @@ _SENSITIVE_HEADERS = frozenset({"authorization", "cookie", "set-cookie", "x-api-key", "x-auth-token"}) +# Hex characters kept from the HMAC. Long enough that collisions are rare across +# an org's traces, short enough to read in a span attribute. +_HASH_HEX_LEN = 8 + +_NON_DIGIT_RE = re.compile(r"[^0-9]") + + +class PiiKind(StrEnum): + """The class of personal identifier a match represents. + + Drives both the visible prefix in the output token and the normalization + applied before hashing, so ``(415) 555-0142`` and ``415-555-0142`` + correlate. + """ + + EMAIL = "email" + PHONE = "phone" + ADDRESS = "address" + + @property + def label(self) -> str: + """The prefix that stays visible in the scrubbed output.""" + return self.value + + def normalize(self, raw: str) -> str: + if self is PiiKind.EMAIL: + return raw.strip().lower() + if self is PiiKind.PHONE: + # Digits only: formatting must not fork the hash. + return _NON_DIGIT_RE.sub("", raw) + # Case-fold and collapse runs of whitespace. + return " ".join(raw.split()).lower() + + +# --- personal identifiers: hashed, not dropped ----------------------------- +# +# These all run after the credential patterns above. +_PERSONAL_PATTERNS: list[tuple[PiiKind, re.Pattern[str]]] = [ + ( + PiiKind.EMAIL, + re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+\b", re.IGNORECASE), + ), + # Phone: optional country code / area code, then the 3-4 local pair. A + # separator is REQUIRED so bare digit runs (ids, timestamps, amounts) don't + # get eaten. + ( + PiiKind.PHONE, + re.compile(r"(?:\+\d{1,3}[ .-]?)?(?:\(\d{3}\)[ .-]?|\b\d{3}[ .-])?\b\d{3}[ .-]\d{4}\b"), + ), + # US-style street address: house number, 1-3 words, a street suffix. + ( + PiiKind.ADDRESS, + re.compile( + r"\b\d{1,6}\s+(?:[A-Za-z0-9.'-]+\s+){0,3}" + r"(?:street|st|avenue|ave|road|rd|boulevard|blvd|lane|ln|drive|dr|court|ct|way|" + r"terrace|ter|place|pl|circle|cir|highway|hwy|parkway|pkwy|square|sq)\b\.?", + re.IGNORECASE, + ), + ), +] + +_key_lock = threading.Lock() +_pii_hash_key: bytes | None = None + + +def set_pii_hash_key(key: bytes | str) -> bool: + """Install the process-wide HMAC key used to hash personal identifiers. + + Idempotent and set-once: returns ``False`` if a key was already installed + (the existing key is kept — a mid-process rotation would silently fork every + correlation) or if ``key`` is empty. The bootstrap calls this from + ``SMOOAI_OBSERVABILITY_PII_HASH_KEY``. + """ + global _pii_hash_key + raw = key.encode() if isinstance(key, str) else key + if not raw: + return False + with _key_lock: + if _pii_hash_key is not None: + return False + _pii_hash_key = raw + return True + + +def _current_key() -> bytes | None: + return _pii_hash_key + + +def pii_token(kind: PiiKind, raw: str, org_id: str) -> str: + """Hash one known-personal value into its scrubbed token. + + Produces exactly the token :func:`scrub_string_for_org` would have written, + which is how a UI search box finds stored hashes: hash the typed term with + the same org and match. + + Returns ``[:redacted]`` when no key is installed. + """ + return _token_with_key(kind, raw, org_id, _current_key()) + + +def _token_with_key(kind: PiiKind, raw: str, org_id: str, key: bytes | None) -> str: + if not key: + return f"[{kind.label}:redacted]" + # org_id in the HMAC message IS the per-org salt. The kind is in there too so + # a phone and an address that normalize alike can't collide. + msg = b"\x00".join((org_id.encode(), kind.label.encode(), kind.normalize(raw).encode())) + digest = hmac.new(key, msg, "sha256").hexdigest()[:_HASH_HEX_LEN] + return f"[{kind.label}:{digest}]" + def scrub_string(value: str) -> str: - """Redact known secret shapes from a free-form string.""" + """Scrub a free-form string with no org context. + + Credentials dropped, personal identifiers hashed under the empty org salt. + Prefer :func:`scrub_string_for_org` wherever an org id is in hand, so hashes + can't be correlated across tenants. + """ + return _scrub_with_key(value, "", _current_key()) + + +def scrub_string_for_org(value: str, org_id: str) -> str: + """Scrub a free-form string, salting personal-identifier hashes with ``org_id``.""" + return _scrub_with_key(value, org_id, _current_key()) + + +def _scrub_with_key(value: str, org_id: str, key: bytes | None) -> str: out = _BEARER_RE.sub("Bearer [redacted]", value) out = _PASSWORD_RE.sub("password=[redacted]", out) out = _TOKEN_RE.sub(lambda m: f"{m.group('key')}[redacted]", out) out = _SK_RE.sub("sk-[redacted]", out) + for kind, pattern in _PERSONAL_PATTERNS: + out = pattern.sub(lambda m, k=kind: _token_with_key(k, m.group(0), org_id, key), out) return out @@ -46,9 +196,17 @@ def scrub_headers( headers: dict[str, str] | None, ) -> dict[str, str] | None: """Redact sensitive headers wholesale; scrub remaining header values.""" + return scrub_headers_for_org(headers, "") + + +def scrub_headers_for_org( + headers: dict[str, str] | None, + org_id: str, +) -> dict[str, str] | None: + """:func:`scrub_headers` with an org salt for the personal-identifier hashes.""" if not headers: return headers out: dict[str, str] = {} for k, v in headers.items(): - out[k] = "[redacted]" if k.lower() in _SENSITIVE_HEADERS else scrub_string(v) + out[k] = "[redacted]" if k.lower() in _SENSITIVE_HEADERS else scrub_string_for_org(v, org_id) return out diff --git a/python/tests/test_pii.py b/python/tests/test_pii.py index 3f256ea..5c13bf3 100644 --- a/python/tests/test_pii.py +++ b/python/tests/test_pii.py @@ -1,3 +1,5 @@ +import pytest + from smooai_observability import pii @@ -46,3 +48,164 @@ def test_scrub_headers_none_passthrough(): def test_scrub_headers_scrubs_nonsensitive_values(): out = pii.scrub_headers({"X-Note": "Bearer leaked.token"}) assert out["X-Note"] == "Bearer [redacted]" + + +# ---- PII hashing parity with rust/observability/src/pii.rs ----------------- +# +# These drive the private _scrub_with_key / _token_with_key rather than +# installing the process-wide key: set_pii_hash_key is set-once and pytest runs +# the suite in one process, so a global write would make it order-dependent. + +KEY = b"test-hmac-key-not-a-real-secret" +OTHER_KEY = b"a-different-test-hmac-key" + + +def scrub(value: str, org: str = "org-1") -> str: + return pii._scrub_with_key(value, org, KEY) + + +def test_credentials_are_dropped_not_hashed(): + # A live token must never become a correlatable handle — hashing one still + # yields an oracle you can test candidate tokens against. + out = scrub("Bearer abc.def-ghi_123 password=hunter2 sk-ABCDEFGHIJKLMNOPQRSTUVWX") + assert "Bearer [redacted]" in out + assert "password=[redacted]" in out + assert "sk-[redacted]" in out + for shape in ("[token:", "[credential:", "[email:", "[phone:"): + assert shape not in out, out + # And a personal identifier hiding inside a secret goes with it. + out2 = scrub("token=a@b.com") + assert "a@b.com" not in out2 + assert "[email:" not in out2 + + +def test_hashes_emails_keeping_the_type_prefix(): + out = scrub("contact me at Alice@Example.com please") + assert "alice@example.com" not in out.lower() + assert out.startswith("contact me at [email:") + assert out.endswith("] please") + + +def test_hashes_phone_numbers(): + for raw in ("555-0142", "(415) 555-0142", "+1 415-555-0142"): + out = scrub(f"call {raw} today") + assert "[phone:" in out, out + assert "0142" not in out, out + + +def test_hashes_street_addresses(): + out = scrub("ship to 1600 Pennsylvania Ave, Washington") + assert "[address:" in out, out + assert "Pennsylvania" not in out, out + + +def test_same_value_same_org_is_stable(): + assert scrub("a@b.com") == scrub("a@b.com") + # …and correlation survives formatting differences in phones. + assert scrub("(415) 555-0142") == scrub("415-555-0142") + + +def test_same_value_different_org_hashes_differently(): + a = scrub("a@b.com", "org-1") + b = scrub("a@b.com", "org-2") + assert a != b, f"per-org salt missing: {a} == {b}" + assert a.startswith("[email:") and b.startswith("[email:") + + +def test_different_key_hashes_differently(): + a = pii._scrub_with_key("a@b.com", "org-1", KEY) + b = pii._scrub_with_key("a@b.com", "org-1", OTHER_KEY) + assert a != b, f"hash is not keyed: {a} == {b}" + + +def test_no_key_redacts_rather_than_hashing(): + # Fail safe: an unkeyed digest of an email is rainbow-tabled instantly. + out = pii._scrub_with_key("a@b.com and 555-0142", "org-1", None) + assert "[email:redacted]" in out, out + assert "[phone:redacted]" in out, out + assert "a@b.com" not in out + assert "0142" not in out + + +def test_hash_is_short_hex(): + token = pii._token_with_key(pii.PiiKind.EMAIL, "a@b.com", "org-1", KEY) + digest = token.removeprefix("[email:").removesuffix("]") + assert len(digest) == pii._HASH_HEX_LEN, token + assert all(c in "0123456789abcdef" for c in digest), token + + +def test_pii_token_matches_what_scrubbing_wrote(): + # The searchability contract: hashing a typed query term must produce + # exactly the token stored in the span. + scrubbed = pii._scrub_with_key("mail a@b.com now", "org-7", KEY) + term = pii._token_with_key(pii.PiiKind.EMAIL, "A@B.com ", "org-7", KEY) + assert term in scrubbed, f"{scrubbed} vs {term}" + + +def test_ordinary_numbers_are_not_mistaken_for_phones(): + # The phone pattern requires a separator before the last 4 digits; ids, + # versions, dates and amounts must survive intact. + for value in ( + "took 1234 ms", + "version 1.2.3", + "2026-08-15T12:00:00Z", + "total 1234.5678", + "trace 0af7651916cd43dd8448eb211c80319c", + ): + assert scrub(value) == value, f"false positive on {value}" + + +def test_set_pii_hash_key_is_set_once_and_rejects_empty(): + assert pii.set_pii_hash_key(b"") is False + saved = pii._pii_hash_key + pii._pii_hash_key = None + try: + assert pii.set_pii_hash_key("first-key") is True + assert pii.set_pii_hash_key("second-key") is False, "rotation should be refused" + assert pii._pii_hash_key == b"first-key" + finally: + pii._pii_hash_key = saved + + +def test_scrub_headers_for_org_never_leaks_raw_email(): + out = pii.scrub_headers_for_org({"X-Note": "reply to a@b.com"}, "org-1") + assert "a@b.com" not in out["X-Note"] + assert "[email:" in out["X-Note"] + + +def test_bootstrap_reads_pii_hash_key_from_env(monkeypatch): + from smooai_observability import bootstrap as bootstrap_mod + + monkeypatch.setenv("SMOOAI_OBSERVABILITY_PII_HASH_KEY", "env-supplied-key") + monkeypatch.setenv("SMOOAI_OBSERVABILITY_DISABLED", "1") + monkeypatch.setattr(bootstrap_mod, "_bootstrapped", None) + saved = pii._pii_hash_key + pii._pii_hash_key = None + try: + bootstrap_mod.bootstrap_observability() + assert pii._pii_hash_key == b"env-supplied-key" + finally: + pii._pii_hash_key = saved + bootstrap_mod._bootstrapped = None + + +@pytest.mark.parametrize( + ("kind", "raw", "org", "expected"), + [ + (pii.PiiKind.EMAIL, "a@b.com", "org-1", "[email:02ea437f]"), + (pii.PiiKind.EMAIL, "A@B.COM ", "org-1", "[email:02ea437f]"), + (pii.PiiKind.EMAIL, "a@b.com", "org-2", "[email:fd96f7dc]"), + (pii.PiiKind.EMAIL, "a@b.com", "", "[email:453b154f]"), + (pii.PiiKind.PHONE, "(415) 555-0142", "org-1", "[phone:415a9aea]"), + (pii.PiiKind.PHONE, "415-555-0142", "org-1", "[phone:415a9aea]"), + (pii.PiiKind.ADDRESS, "1600 Pennsylvania Ave", "org-1", "[address:c5351f4a]"), + ], +) +def test_cross_sdk_parity_vectors(kind, raw, org, expected): + """Pins the exact bytes every SDK must produce. + + Computed independently and asserted verbatim in all five SDKs. If any SDK's + message framing, normalization or truncation drifts, exactly one of these + breaks. + """ + assert pii._token_with_key(kind, raw, org, KEY) == expected diff --git a/rust/observability/README.md b/rust/observability/README.md index bd86d60..a7bda2d 100644 --- a/rust/observability/README.md +++ b/rust/observability/README.md @@ -19,7 +19,7 @@ error-safe and degrades to a no-op (plus one stderr line) rather than panicking. | Scope / context (per-task) | `scope.ts` | ✅ | | Breadcrumb buffer (max 100) | `scope.ts` | ✅ | | PII scrubbing (credentials) | `pii.ts` | ✅ | -| PII **hashing** (email/phone/addr) | — (Rust only) | ✅ | +| PII **hashing** (email/phone/addr) | `pii.ts` (all 5 SDKs) | ✅ | | Batched webhook transport + retry | `transport.ts` | ✅ | | OTLP traces + metrics export | `otel/setup-otel-sdk.ts` | ✅ | | Per-request M2M auth (no staleness)| `otel/auth-injecting-exporter.ts` | ✅ | @@ -91,6 +91,10 @@ Set the key with `SMOOAI_OBSERVABILITY_PII_HASH_KEY` (read by `bootstrap()`) or `pii::set_pii_hash_key`. **With no key, personal identifiers are fully redacted** (`[email:redacted]`) rather than hashed under a guessable one. +The TypeScript, Go, Python and .NET SDKs implement the identical scheme and emit +byte-identical tokens; the shared vectors are asserted in every SDK's PII test +suite (`cross_sdk_parity_vectors`). + ⚠️ **The key and the org id are load-bearing.** Rotating either silently breaks correlation with every hash already stored — treat the key as permanent, and do not reuse a secret that rotates on a schedule. diff --git a/rust/observability/src/bootstrap.rs b/rust/observability/src/bootstrap.rs index 0d31c8e..2535465 100644 --- a/rust/observability/src/bootstrap.rs +++ b/rust/observability/src/bootstrap.rs @@ -243,7 +243,6 @@ async fn build(env: BootstrapEnv) -> BootstrapResult { } } - #[cfg(test)] mod exporting_status_tests { use super::*; diff --git a/rust/observability/src/pii.rs b/rust/observability/src/pii.rs index f856bf7..99b53d3 100644 --- a/rust/observability/src/pii.rs +++ b/rust/observability/src/pii.rs @@ -29,8 +29,9 @@ //! in `before_send`. //! //! Credential patterns are a direct port of the TS `pii.ts` patterns. The -//! hashing layer is Rust-only today — the other four SDKs still scrub -//! credentials only. +//! hashing layer is implemented identically in all five SDKs (TypeScript, Rust, +//! Go, Python, .NET) — see `cross_sdk_parity_vectors` for the shared vectors +//! every port asserts verbatim. use once_cell::sync::{Lazy, OnceCell}; use regex::Regex; @@ -411,6 +412,40 @@ mod tests { assert!(hex.chars().all(|c| c.is_ascii_hexdigit()), "{t}"); } + /// Pins the exact bytes every SDK must produce. Computed independently + /// (python `hmac.new(key, org\0kind\0normalized, "sha256")`) and asserted + /// verbatim in the TS, Go, Python and .NET ports too. If any SDK's message + /// framing, normalization or truncation drifts, exactly one of these breaks. + #[test] + fn cross_sdk_parity_vectors() { + let cases = [ + (PiiKind::Email, "a@b.com", "org-1", "[email:02ea437f]"), + (PiiKind::Email, "A@B.COM ", "org-1", "[email:02ea437f]"), + (PiiKind::Email, "a@b.com", "org-2", "[email:fd96f7dc]"), + (PiiKind::Email, "a@b.com", "", "[email:453b154f]"), + ( + PiiKind::Phone, + "(415) 555-0142", + "org-1", + "[phone:415a9aea]", + ), + (PiiKind::Phone, "415-555-0142", "org-1", "[phone:415a9aea]"), + ( + PiiKind::Address, + "1600 Pennsylvania Ave", + "org-1", + "[address:c5351f4a]", + ), + ]; + for (kind, raw, org, want) in cases { + assert_eq!( + token_with_key(kind, raw, org, Some(KEY)), + want, + "{kind:?} {raw:?} {org:?}" + ); + } + } + #[test] fn pii_token_matches_what_scrubbing_wrote() { // The searchability contract: hashing a typed query term must produce