diff --git a/sdk/experimental/tdf/doc.go b/sdk/experimental/tdf/doc.go index 8e80e36173..798138dcad 100644 --- a/sdk/experimental/tdf/doc.go +++ b/sdk/experimental/tdf/doc.go @@ -28,7 +28,6 @@ // if err != nil { // log.Fatal(err) // } -// defer writer.Close() // // // Write data segments (can be out-of-order) // data1 := []byte("First segment") @@ -44,15 +43,18 @@ // } // // // Finalize with attributes and options -// finalBytes, manifest, err := writer.Finalize(ctx, -// WithAttributeValues(attributes), -// WithPayloadMimeType("text/plain"), -// WithEncryptedMetadata("sensitive metadata"), +// result, err := writer.Finalize(ctx, +// tdf.WithAttributeValues(attributes), +// tdf.WithPayloadMimeType("text/plain"), +// tdf.WithEncryptedMetadata("sensitive metadata"), // ) // if err != nil { // log.Fatal(err) // } // +// // result.Data holds the archive's closing bytes; append it after each +// // segment's SegmentResult.TDFData, in ascending segment index order. +// // # Initial Attributes and Default KAS at Writer Creation // // Callers can provide initial attributes and a default KAS when constructing @@ -70,26 +72,32 @@ // } // // Later, Finalize without attributes/KAS uses the initial values. // -// # Segment Overrides at Finalize (Contiguous Prefix) +// # Segment Overrides at Finalize +// +// By default Finalize describes every written segment, ordered by index. +// WithSegments narrows that to a chosen subset. // -// You can restrict finalization to a contiguous prefix of written segments -// using `WithSegments([]int{0, 1, ..., K})`. Indices must start at 0 with no -// gaps or duplicates, and no segments may have been written beyond K. +// Indices need not be contiguous — a caller mapping S3 multipart uploads +// onto segments might write 0, 1, 5000, 5001 — but the list must name +// written segments in ascending index order and may drop only from the end. +// That is the order a reader concatenates the payload in, so dropping a +// segment from the middle would shift every later segment's offset and +// produce an unreadable TDF. // // // Write segments 0 and 1 // _, _ = writer.WriteSegment(ctx, 0, []byte("part-0")) // _, _ = writer.WriteSegment(ctx, 1, []byte("part-1")) // -// // Finalize keeping the prefix [0,1] -// finalBytes, manifest, err := writer.Finalize(ctx, -// tdf.WithSegments([]int{0, 1}), +// // Finalize keeping only segment 0 +// result, err := writer.Finalize(ctx, +// tdf.WithSegments([]int{0}), // ) // if err != nil { // log.Fatal(err) // } // -// If all segments should be kept, `WithSegments([0..N-1])` is equivalent to -// the default behavior and is optional. +// Keeping every written segment is the default, so passing the full list is +// optional. // // # Advanced Features // @@ -103,19 +111,28 @@ // // # Architecture // -// The TDF writer uses a two-layer architecture: +// The TDF writer uses a three-layer architecture: // -// 1. TDF Layer (tdf.Writer): Handles encryption, assertions, and TDF protocol logic -// 2. Archive Layer (internal/zipstream): Manages ZIP file structure and segment assembly +// 1. Adapter Layer (tdf.Writer): Maps this package's options onto the stable +// writer and supplies multi-KAS ABAC key splitting +// 2. TDF Layer (sdk.ChunkedWriter): Handles encryption, assertions, and TDF +// protocol logic +// 3. Archive Layer (internal/zipstream): Manages ZIP file structure and +// segment assembly // // This separation enables independent optimization of cryptographic operations -// and file format handling. +// and file format handling. Callers who need only a single KAS can skip layer +// one and use [github.com/opentdf/platform/sdk.NewChunkedWriter] directly. // // # Thread Safety // -// Writers are safe for concurrent use with proper external synchronization. -// Individual WriteSegment calls must be serialized, but multiple writers -// can operate independently. +// A Writer is safe for concurrent use. [Writer.WriteSegment] may be called +// from several goroutines at once so long as each targets a distinct segment +// index; two concurrent calls for the same index are not allowed, and one of +// them will fail rather than corrupt the archive. +// +// Finalize is terminal and takes an exclusive lock, so it must not overlap +// with any in-flight WriteSegment call. // // # Performance Characteristics // diff --git a/sdk/experimental/tdf/key_access.go b/sdk/experimental/tdf/key_access.go deleted file mode 100644 index ad849c739c..0000000000 --- a/sdk/experimental/tdf/key_access.go +++ /dev/null @@ -1,266 +0,0 @@ -// Experimental: This package is EXPERIMENTAL and may change or be removed at any time - -package tdf - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "log/slog" - - "github.com/opentdf/platform/lib/ocrypto" - "github.com/opentdf/platform/sdk/experimental/tdf/keysplit" -) - -var tdfSaltBytes []byte - -// tdfSalt generates the standard TDF salt for key derivation -func init() { - digest := sha256.New() - digest.Write([]byte("TDF")) - tdfSaltBytes = digest.Sum(nil) -} - -func tdfSalt() []byte { - return tdfSaltBytes -} - -// BuildKeyAccessObjects creates KeyAccess objects from splits for TDF manifest inclusion -func buildKeyAccessObjects(result *keysplit.SplitResult, policyBytes []byte, metadata string) ([]KeyAccess, error) { - if result == nil || len(result.Splits) == 0 { - return nil, errors.New("no splits provided") - } - - var keyAccessList []KeyAccess - - // Create base64-encoded policy for binding - base64Policy := string(ocrypto.Base64Encode(policyBytes)) - - for _, split := range result.Splits { - for _, kasURL := range split.KASURLs { - // Get public key info for this KAS - pubKeyInfo, exists := result.KASPublicKeys[kasURL] - if !exists { - slog.Warn("no public key found for KAS, skipping", - slog.String("kas_url", kasURL), - slog.String("split_id", split.ID)) - continue - } - - // Create policy binding - policyBinding := createPolicyBinding(split.Data, base64Policy) - - // Encrypt metadata if provided - var encryptedMetadata string - if metadata != "" { - var err error - encryptedMetadata, err = encryptMetadata(split.Data, metadata) - if err != nil { - return nil, fmt.Errorf("failed to encrypt metadata for KAS %s: %w", kasURL, err) - } - } - - // Encrypt the split key with KAS public key - wrappedKey, keyType, ephemeralPubKey, err := wrapKeyWithPublicKey(split.Data, pubKeyInfo) - if err != nil { - return nil, fmt.Errorf("failed to wrap key for KAS %s: %w", kasURL, err) - } - - // Build the KeyAccess object - keyAccess := KeyAccess{ - KeyType: keyType, - KasURL: kasURL, - KID: pubKeyInfo.KID, - Protocol: "kas", - SplitID: split.ID, - WrappedKey: wrappedKey, - PolicyBinding: policyBinding, - EncryptedMetadata: encryptedMetadata, - } - - // Add ephemeral public key for EC keys - if ephemeralPubKey != "" { - keyAccess.EphemeralPublicKey = ephemeralPubKey - } - - keyAccessList = append(keyAccessList, keyAccess) - - slog.Debug("created key access object", - slog.String("kas_url", kasURL), - slog.String("split_id", split.ID), - slog.String("key_type", keyType), - slog.String("kid", pubKeyInfo.KID)) - } - } - - if len(keyAccessList) == 0 { - return nil, errors.New("no valid key access objects generated") - } - - slog.Debug("built key access objects", - slog.Int("num_key_access", len(keyAccessList)), - slog.Int("num_splits", len(result.Splits))) - - return keyAccessList, nil -} - -// createPolicyBinding creates an HMAC binding between the key and policy -func createPolicyBinding(symKey []byte, base64PolicyObject string) any { - // Create HMAC hash of the policy using the symmetric key - hmacHash := ocrypto.CalculateSHA256Hmac(symKey, []byte(base64PolicyObject)) - - // Convert to hex string - hashHex := hex.EncodeToString(hmacHash) - - // Create policy binding structure - binding := PolicyBinding{ - Alg: kPolicyBindingAlg, - Hash: string(ocrypto.Base64Encode([]byte(hashHex))), - } - - // Return as any to match KeyAccess.PolicyBinding field - return binding -} - -// encryptMetadata encrypts TDF metadata using the split key -func encryptMetadata(symKey []byte, metadata string) (string, error) { - // Create AES-GCM cipher - gcm, err := ocrypto.NewAESGcm(symKey) - if err != nil { - return "", fmt.Errorf("failed to create AES-GCM: %w", err) - } - - // Encrypt the metadata - encryptedBytes, err := gcm.Encrypt([]byte(metadata)) - if err != nil { - return "", fmt.Errorf("failed to encrypt metadata: %w", err) - } - - // Extract IV (first 12 bytes for GCM) - iv := encryptedBytes[:ocrypto.GcmStandardNonceSize] - - // Create encrypted metadata structure - encMeta := EncryptedMetadata{ - Cipher: string(ocrypto.Base64Encode(encryptedBytes)), - Iv: string(ocrypto.Base64Encode(iv)), - } - - // Serialize to JSON and base64 encode - metadataJSON, err := json.Marshal(encMeta) - if err != nil { - return "", fmt.Errorf("failed to marshal encrypted metadata: %w", err) - } - - return string(ocrypto.Base64Encode(metadataJSON)), nil -} - -// wrapKeyWithPublicKey encrypts a symmetric key with a KAS public key -func wrapKeyWithPublicKey(symKey []byte, pubKeyInfo keysplit.KASPublicKey) (string, string, string, error) { - if pubKeyInfo.PEM == "" { - return "", "", "", fmt.Errorf("public key PEM is empty for KAS %s", pubKeyInfo.URL) - } - - // Determine key type based on algorithm - ktype := ocrypto.KeyType(pubKeyInfo.Algorithm) - - if ocrypto.IsKEMKeyType(ktype) { - return wrapKeyWithKEM(ktype, pubKeyInfo.PEM, symKey) - } - if ocrypto.IsECKeyType(ktype) { - // Handle EC key wrapping - return wrapKeyWithEC(ktype, pubKeyInfo.PEM, symKey) - } - // Handle RSA key wrapping - wrapped, err := wrapKeyWithRSA(pubKeyInfo.PEM, symKey) - return wrapped, "wrapped", "", err -} - -// wrapKeyWithEC encrypts a key using EC public key with ECIES -func wrapKeyWithEC(keyType ocrypto.KeyType, kasPublicKeyPEM string, symKey []byte) (string, string, string, error) { - // Convert key type to ECC mode - mode, err := ocrypto.ECKeyTypeToMode(keyType) - if err != nil { - return "", "", "", fmt.Errorf("failed to convert key type to ECC mode: %w", err) - } - - // Generate ephemeral key pair - ecKeyPair, err := ocrypto.NewECKeyPair(mode) - if err != nil { - return "", "", "", fmt.Errorf("failed to create EC key pair: %w", err) - } - - // Get ephemeral public key in PEM format - ephemeralPubKey, err := ecKeyPair.PublicKeyInPemFormat() - if err != nil { - return "", "", "", fmt.Errorf("failed to get ephemeral public key: %w", err) - } - - // Get ephemeral private key - ephemeralPrivKey, err := ecKeyPair.PrivateKeyInPemFormat() - if err != nil { - return "", "", "", fmt.Errorf("failed to get ephemeral private key: %w", err) - } - - // Compute ECDH shared secret - ecdhKey, err := ocrypto.ComputeECDHKey([]byte(ephemeralPrivKey), []byte(kasPublicKeyPEM)) - if err != nil { - return "", "", "", fmt.Errorf("failed to compute ECDH key: %w", err) - } - - // Derive wrapping key using HKDF - salt := tdfSalt() - wrapKey, err := ocrypto.CalculateHKDF(salt, ecdhKey) - if err != nil { - return "", "", "", fmt.Errorf("failed to derive wrap key: %w", err) - } - - // Ensure we have the right length for wrapping, trim if needed, or error if too short - if len(wrapKey) > len(symKey) { - wrapKey = wrapKey[:len(symKey)] - } else if len(wrapKey) < len(symKey) { - return "", "", "", fmt.Errorf("wrap key too short: got %d, expected at least %d", - len(wrapKey), len(symKey)) - } - - wrapped := make([]byte, len(symKey)) - for i := range symKey { - wrapped[i] = symKey[i] ^ wrapKey[i] - } - - return string(ocrypto.Base64Encode(wrapped)), "eccWrapped", ephemeralPubKey, nil -} - -// wrapKeyWithRSA encrypts a key using RSA public key with OAEP padding -func wrapKeyWithRSA(kasPublicKeyPEM string, symKey []byte) (string, error) { - // Create RSA encryptor from PEM - encryptor, err := ocrypto.FromPublicPEM(kasPublicKeyPEM) - if err != nil { - return "", fmt.Errorf("failed to create RSA encryptor: %w", err) - } - - // Encrypt with OAEP padding - encryptedKey, err := encryptor.Encrypt(symKey) - if err != nil { - return "", fmt.Errorf("failed to RSA encrypt key: %w", err) - } - - return string(ocrypto.Base64Encode(encryptedKey)), nil -} - -// wrapKeyWithKEM wraps a DEK with any KEM scheme — pure ML-KEM or hybrid -// (X-Wing, NIST PQ/T). Returns the base64-encoded envelope, the manifest -// scheme name (`hybrid-wrapped` or `mlkem-wrapped`), and an empty ephemeral -// key string (KEMs do not emit one in this profile). -func wrapKeyWithKEM(ktype ocrypto.KeyType, kasPublicKeyPEM string, symKey []byte) (string, string, string, error) { - wrappedDER, err := ocrypto.WrapDEK(ktype, kasPublicKeyPEM, symKey) - if err != nil { - return "", "", "", fmt.Errorf("kem wrap failed: %w", err) - } - scheme := "hybrid-wrapped" - if ocrypto.IsMLKEMKeyType(ktype) { - scheme = "mlkem-wrapped" - } - return string(ocrypto.Base64Encode(wrappedDER)), scheme, "", nil -} diff --git a/sdk/experimental/tdf/key_access_test.go b/sdk/experimental/tdf/key_access_test.go deleted file mode 100644 index 6deac12f27..0000000000 --- a/sdk/experimental/tdf/key_access_test.go +++ /dev/null @@ -1,652 +0,0 @@ -// Experimental: This package is EXPERIMENTAL and may change or be removed at any time - -package tdf - -import ( - "crypto/rand" - "encoding/json" - "strings" - "testing" - - "github.com/opentdf/platform/lib/ocrypto" - "github.com/opentdf/platform/sdk/experimental/tdf/keysplit" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// Test constants for key access operations -const ( - testKAS1URL = "https://kas1.example.com/" - testKAS2URL = "https://kas2.example.com/" - - // Real RSA-2048 public keys for testing key wrapping - testRSAPublicKey = `-----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtQ2ZuyT/p32SFmWTj+wQ -huQwR4IJSzlJ7CqZ4fOXw90rA2joK27dIGiHrtkQHGhS4SK1mvkYyJaREoppMFRc -AyZWCgixbSdwYJS/KN0hjLIdhtkdBlZDaZN2ayTf2sZjWzOLL2cYzzVsAy9tGL8a -bMqf91DEHv+l58fPxmbJ/i6YFFQoOEsyWnPhXdiExe6poQDCHJFYYOp6iu5kOPWr -jKFj9eGXuFR/CJQ/uxTSM+8/7Ejmi8Oa52TQAUhMPH0U1CRFm/NuiFoFissa0jJC -J3k6syxvf45mPrbtlhcELskXrquDtJOpIMQmEwfuV4j8iLNwVlsR2tAbClJi6UOy -SQIDAQAB ------END PUBLIC KEY-----` - - testMetadata = "test metadata content" - testPolicyJSON = `{"uuid":"test","body":{"dataAttributes":[{"attribute":"test"}],"dissem":[]}}` -) - -// createTestSplitResult creates a mock SplitResult for testing key access operations -func createTestSplitResult(pubKey string, algorithm string) *keysplit.SplitResult { - // Generate random split data - splitData := make([]byte, 32) - _, err := rand.Read(splitData) - if err != nil { - panic("failed to generate test split data: " + err.Error()) - } - - split := keysplit.Split{ - ID: "test-split-1", - Data: splitData, - KASURLs: []string{testKAS1URL}, - } - - pubKeyInfo := keysplit.KASPublicKey{ - URL: testKAS1URL, - Algorithm: algorithm, - KID: "test-kid-1", - PEM: pubKey, - } - - return &keysplit.SplitResult{ - Splits: []keysplit.Split{split}, - KASPublicKeys: map[string]keysplit.KASPublicKey{testKAS1URL: pubKeyInfo}, - } -} - -func TestBuildKeyAccessObjects(t *testing.T) { - t.Run("successfully creates key access objects with RSA public key", func(t *testing.T) { - // Test that buildKeyAccessObjects correctly processes RSA keys and creates valid KeyAccess objects - splitResult := createTestSplitResult(testRSAPublicKey, "rsa:2048") - policyBytes := []byte(testPolicyJSON) - metadata := testMetadata - - keyAccessList, err := buildKeyAccessObjects(splitResult, policyBytes, metadata) - - require.NoError(t, err, "Should successfully create key access objects with valid RSA key") - require.Len(t, keyAccessList, 1, "Should create exactly one key access object") - - keyAccess := keyAccessList[0] - assert.Equal(t, "wrapped", keyAccess.KeyType, "RSA keys should use 'wrapped' key type") - assert.Equal(t, testKAS1URL, keyAccess.KasURL, "Should preserve KAS URL") - assert.Equal(t, "test-kid-1", keyAccess.KID, "Should preserve key ID") - assert.Equal(t, "kas", keyAccess.Protocol, "Should use 'kas' protocol") - assert.Equal(t, "test-split-1", keyAccess.SplitID, "Should preserve split ID") - assert.NotEmpty(t, keyAccess.WrappedKey, "Should contain wrapped key data") - assert.NotEmpty(t, keyAccess.PolicyBinding, "Should contain policy binding") - assert.NotEmpty(t, keyAccess.EncryptedMetadata, "Should contain encrypted metadata") - assert.Empty(t, keyAccess.EphemeralPublicKey, "RSA keys should not have ephemeral public key") - }) - - t.Run("successfully creates key access objects with EC public key", func(t *testing.T) { - // Test that buildKeyAccessObjects correctly handles elliptic curve keys with ephemeral key generation - - // Generate a real EC P-256 key pair for testing - ecKeyPair, err := ocrypto.NewECKeyPair(ocrypto.ECCModeSecp256r1) - require.NoError(t, err, "Should generate EC key pair") - - ecPublicKeyPEM, err := ecKeyPair.PublicKeyInPemFormat() - require.NoError(t, err, "Should get public key in PEM format") - - splitResult := createTestSplitResult(ecPublicKeyPEM, "ec:secp256r1") - policyBytes := []byte(testPolicyJSON) - metadata := testMetadata - - keyAccessList, err := buildKeyAccessObjects(splitResult, policyBytes, metadata) - - require.NoError(t, err, "Should successfully create key access objects with valid EC key") - require.Len(t, keyAccessList, 1, "Should create exactly one key access object") - - keyAccess := keyAccessList[0] - assert.Equal(t, "eccWrapped", keyAccess.KeyType, "EC keys should use 'eccWrapped' key type") - assert.Equal(t, testKAS1URL, keyAccess.KasURL, "Should preserve KAS URL") - assert.NotEmpty(t, keyAccess.EphemeralPublicKey, "EC keys should have ephemeral public key") - assert.NotEmpty(t, keyAccess.WrappedKey, "Should contain wrapped key data") - }) - - t.Run("successfully creates key access objects with X-Wing public key", func(t *testing.T) { - xwingKeyPair, err := ocrypto.NewXWingKeyPair() - require.NoError(t, err) - - xwingPublicKeyPEM, err := xwingKeyPair.PublicKeyInPemFormat() - require.NoError(t, err) - - splitResult := createTestSplitResult(xwingPublicKeyPEM, string(ocrypto.HybridXWingKey)) - policyBytes := []byte(testPolicyJSON) - - keyAccessList, err := buildKeyAccessObjects(splitResult, policyBytes, testMetadata) - - require.NoError(t, err, "Should successfully create key access objects with valid X-Wing key") - require.Len(t, keyAccessList, 1) - - keyAccess := keyAccessList[0] - assert.Equal(t, "hybrid-wrapped", keyAccess.KeyType) - assert.NotEmpty(t, keyAccess.WrappedKey) - assert.Empty(t, keyAccess.EphemeralPublicKey) - }) - - t.Run("successfully creates key access objects with P256+ML-KEM-768 public key", func(t *testing.T) { - keyPair, err := ocrypto.NewP256MLKEM768KeyPair() - require.NoError(t, err) - - publicKeyPEM, err := keyPair.PublicKeyInPemFormat() - require.NoError(t, err) - - splitResult := createTestSplitResult(publicKeyPEM, string(ocrypto.HybridSecp256r1MLKEM768Key)) - policyBytes := []byte(testPolicyJSON) - - keyAccessList, err := buildKeyAccessObjects(splitResult, policyBytes, testMetadata) - - require.NoError(t, err, "Should successfully create key access objects with valid P256+ML-KEM-768 key") - require.Len(t, keyAccessList, 1) - - keyAccess := keyAccessList[0] - assert.Equal(t, "hybrid-wrapped", keyAccess.KeyType) - assert.NotEmpty(t, keyAccess.WrappedKey) - assert.Empty(t, keyAccess.EphemeralPublicKey) - }) - - t.Run("successfully creates key access objects with P384+ML-KEM-1024 public key", func(t *testing.T) { - keyPair, err := ocrypto.NewP384MLKEM1024KeyPair() - require.NoError(t, err) - - publicKeyPEM, err := keyPair.PublicKeyInPemFormat() - require.NoError(t, err) - - splitResult := createTestSplitResult(publicKeyPEM, string(ocrypto.HybridSecp384r1MLKEM1024Key)) - policyBytes := []byte(testPolicyJSON) - - keyAccessList, err := buildKeyAccessObjects(splitResult, policyBytes, testMetadata) - - require.NoError(t, err, "Should successfully create key access objects with valid P384+ML-KEM-1024 key") - require.Len(t, keyAccessList, 1) - - keyAccess := keyAccessList[0] - assert.Equal(t, "hybrid-wrapped", keyAccess.KeyType) - assert.NotEmpty(t, keyAccess.WrappedKey) - assert.Empty(t, keyAccess.EphemeralPublicKey) - }) - - t.Run("handles multiple KAS URLs in single split", func(t *testing.T) { - // Test that multiple KAS URLs in one split create separate KeyAccess objects - splitData := make([]byte, 32) - _, err := rand.Read(splitData) - require.NoError(t, err) - - split := keysplit.Split{ - ID: "multi-kas-split", - Data: splitData, - KASURLs: []string{testKAS1URL, testKAS2URL}, - } - - splitResult := &keysplit.SplitResult{ - Splits: []keysplit.Split{split}, - KASPublicKeys: map[string]keysplit.KASPublicKey{ - testKAS1URL: {URL: testKAS1URL, Algorithm: "rsa:2048", KID: "kid1", PEM: testRSAPublicKey}, - testKAS2URL: {URL: testKAS2URL, Algorithm: "rsa:2048", KID: "kid2", PEM: testRSAPublicKey}, - }, - } - - keyAccessList, err := buildKeyAccessObjects(splitResult, []byte(testPolicyJSON), "") - - require.NoError(t, err, "Should handle multiple KAS URLs") - assert.Len(t, keyAccessList, 2, "Should create separate KeyAccess for each KAS URL") - - kasURLs := []string{keyAccessList[0].KasURL, keyAccessList[1].KasURL} - assert.Contains(t, kasURLs, testKAS1URL, "Should include first KAS URL") - assert.Contains(t, kasURLs, testKAS2URL, "Should include second KAS URL") - }) - - t.Run("skips KAS URLs without public keys", func(t *testing.T) { - // Test that missing public keys are handled gracefully by skipping those KAS - splitData := make([]byte, 32) - _, err := rand.Read(splitData) - require.NoError(t, err) - - split := keysplit.Split{ - ID: "missing-key-split", - Data: splitData, - KASURLs: []string{testKAS1URL, testKAS2URL}, - } - - // Only provide public key for one KAS - splitResult := &keysplit.SplitResult{ - Splits: []keysplit.Split{split}, - KASPublicKeys: map[string]keysplit.KASPublicKey{ - testKAS1URL: {URL: testKAS1URL, Algorithm: "rsa:2048", KID: "kid1", PEM: testRSAPublicKey}, - // testKAS2URL intentionally missing - }, - } - - keyAccessList, err := buildKeyAccessObjects(splitResult, []byte(testPolicyJSON), "") - - require.NoError(t, err, "Should handle missing public keys gracefully") - assert.Len(t, keyAccessList, 1, "Should create KeyAccess only for KAS with public key") - assert.Equal(t, testKAS1URL, keyAccessList[0].KasURL, "Should use KAS with available public key") - }) - - t.Run("handles empty metadata correctly", func(t *testing.T) { - // Test that empty metadata is handled without creating encrypted metadata - splitResult := createTestSplitResult(testRSAPublicKey, "rsa:2048") - - keyAccessList, err := buildKeyAccessObjects(splitResult, []byte(testPolicyJSON), "") - - require.NoError(t, err, "Should handle empty metadata") - require.Len(t, keyAccessList, 1, "Should create key access object") - assert.Empty(t, keyAccessList[0].EncryptedMetadata, "Should not create encrypted metadata for empty input") - }) - - t.Run("returns error for nil split result", func(t *testing.T) { - // Test error handling for invalid input - _, err := buildKeyAccessObjects(nil, []byte(testPolicyJSON), "") - - require.Error(t, err, "Should return error for nil split result") - assert.Contains(t, err.Error(), "no splits provided", "Error should mention missing splits") - }) - - t.Run("returns error for empty splits", func(t *testing.T) { - // Test error handling for empty splits list - splitResult := &keysplit.SplitResult{ - Splits: []keysplit.Split{}, - KASPublicKeys: map[string]keysplit.KASPublicKey{}, - } - - _, err := buildKeyAccessObjects(splitResult, []byte(testPolicyJSON), "") - - require.Error(t, err, "Should return error for empty splits") - assert.Contains(t, err.Error(), "no splits provided", "Error should mention missing splits") - }) - - t.Run("returns error when no valid key access objects generated", func(t *testing.T) { - // Test error when all KAS URLs lack public keys - splitData := make([]byte, 32) - _, err := rand.Read(splitData) - require.NoError(t, err) - - split := keysplit.Split{ - ID: "no-keys-split", - Data: splitData, - KASURLs: []string{testKAS1URL}, - } - - splitResult := &keysplit.SplitResult{ - Splits: []keysplit.Split{split}, - KASPublicKeys: map[string]keysplit.KASPublicKey{}, // Empty - no public keys - } - - _, err = buildKeyAccessObjects(splitResult, []byte(testPolicyJSON), "") - - require.Error(t, err, "Should return error when no key access objects can be generated") - assert.Contains(t, err.Error(), "no valid key access objects generated", "Error should mention no valid objects") - }) -} - -func TestCreatePolicyBinding(t *testing.T) { - t.Run("creates consistent HMAC policy binding", func(t *testing.T) { - // Test that policy binding creates consistent HMAC hash - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - base64Policy := string(ocrypto.Base64Encode([]byte(testPolicyJSON))) - - binding := createPolicyBinding(symKey, base64Policy) - require.IsType(t, PolicyBinding{}, binding, "Should return PolicyBinding type") - - policyBinding, ok := binding.(PolicyBinding) - require.True(t, ok, "Policy binding should be PolicyBinding type") - assert.Equal(t, "HS256", policyBinding.Alg, "Should use HS256 algorithm") - assert.NotEmpty(t, policyBinding.Hash, "Should contain hash value") - - // Verify hash is base64 encoded - _, err = ocrypto.Base64Decode([]byte(policyBinding.Hash)) - require.NoError(t, err, "Hash should be valid base64") - }) - - t.Run("produces different hashes for different policies", func(t *testing.T) { - // Test that different policies produce different bindings - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - policy1 := string(ocrypto.Base64Encode([]byte(`{"policy": "test1"}`))) - policy2 := string(ocrypto.Base64Encode([]byte(`{"policy": "test2"}`))) - - binding1 := createPolicyBinding(symKey, policy1) - binding2 := createPolicyBinding(symKey, policy2) - - pb1, ok1 := binding1.(PolicyBinding) - require.True(t, ok1, "binding1 should be PolicyBinding type") - pb2, ok2 := binding2.(PolicyBinding) - require.True(t, ok2, "binding2 should be PolicyBinding type") - hash1 := pb1.Hash - hash2 := pb2.Hash - assert.NotEqual(t, hash1, hash2, "Different policies should produce different hashes") - }) - - t.Run("produces different hashes for different keys", func(t *testing.T) { - // Test that different symmetric keys produce different bindings - symKey1 := make([]byte, 32) - symKey2 := make([]byte, 32) - _, err := rand.Read(symKey1) - require.NoError(t, err) - _, err = rand.Read(symKey2) - require.NoError(t, err) - - policy := string(ocrypto.Base64Encode([]byte(testPolicyJSON))) - - binding1 := createPolicyBinding(symKey1, policy) - binding2 := createPolicyBinding(symKey2, policy) - - pb1, ok1 := binding1.(PolicyBinding) - require.True(t, ok1, "binding1 should be PolicyBinding type") - pb2, ok2 := binding2.(PolicyBinding) - require.True(t, ok2, "binding2 should be PolicyBinding type") - hash1 := pb1.Hash - hash2 := pb2.Hash - assert.NotEqual(t, hash1, hash2, "Different keys should produce different hashes") - }) -} - -func TestEncryptMetadata(t *testing.T) { - t.Run("encrypts metadata using AES-GCM", func(t *testing.T) { - // Test successful metadata encryption with proper structure - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - encryptedMetadata, err := encryptMetadata(symKey, testMetadata) - - require.NoError(t, err, "Should encrypt metadata successfully") - assert.NotEmpty(t, encryptedMetadata, "Should return encrypted metadata") - - // Verify it's base64 encoded - decodedJSON, err := ocrypto.Base64Decode([]byte(encryptedMetadata)) - require.NoError(t, err, "Encrypted metadata should be valid base64") - - // Verify JSON structure - var encMeta EncryptedMetadata - err = json.Unmarshal(decodedJSON, &encMeta) - require.NoError(t, err, "Should unmarshal to EncryptedMetadata structure") - - assert.NotEmpty(t, encMeta.Cipher, "Should contain cipher text") - assert.NotEmpty(t, encMeta.Iv, "Should contain IV") - - // Verify IV and cipher are base64 - _, err = ocrypto.Base64Decode([]byte(encMeta.Iv)) - require.NoError(t, err, "IV should be valid base64") - _, err = ocrypto.Base64Decode([]byte(encMeta.Cipher)) - require.NoError(t, err, "Cipher should be valid base64") - }) - - t.Run("produces different ciphertext for same metadata with different keys", func(t *testing.T) { - // Test that different keys produce different encrypted output - symKey1 := make([]byte, 32) - symKey2 := make([]byte, 32) - _, err := rand.Read(symKey1) - require.NoError(t, err) - _, err = rand.Read(symKey2) - require.NoError(t, err) - - encrypted1, err1 := encryptMetadata(symKey1, testMetadata) - encrypted2, err2 := encryptMetadata(symKey2, testMetadata) - - require.NoError(t, err1, "Should encrypt with first key") - require.NoError(t, err2, "Should encrypt with second key") - assert.NotEqual(t, encrypted1, encrypted2, "Different keys should produce different ciphertext") - }) - - t.Run("handles empty metadata", func(t *testing.T) { - // Test encryption of empty string - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - encryptedMetadata, err := encryptMetadata(symKey, "") - - require.NoError(t, err, "Should handle empty metadata") - assert.NotEmpty(t, encryptedMetadata, "Should still return encrypted structure") - }) - - t.Run("returns error for invalid key size", func(t *testing.T) { - // Test error handling for incorrect key size (empty key) - emptyKey := make([]byte, 0) - - _, err := encryptMetadata(emptyKey, testMetadata) - - require.Error(t, err, "Should return error for empty key") - assert.Contains(t, err.Error(), "AES-GCM", "Error should mention AES-GCM creation failure") - }) -} - -func TestWrapKeyWithPublicKey(t *testing.T) { - t.Run("wraps key with RSA public key", func(t *testing.T) { - // Test RSA key wrapping functionality - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - pubKeyInfo := keysplit.KASPublicKey{ - URL: testKAS1URL, - Algorithm: "rsa:2048", - KID: "test-kid", - PEM: testRSAPublicKey, - } - - wrappedKey, keyType, ephemeralPubKey, err := wrapKeyWithPublicKey(symKey, pubKeyInfo) - - require.NoError(t, err, "Should wrap key with RSA public key") - assert.NotEmpty(t, wrappedKey, "Should return wrapped key") - assert.Equal(t, "wrapped", keyType, "RSA keys should use 'wrapped' type") - assert.Empty(t, ephemeralPubKey, "RSA should not generate ephemeral public key") - - // Verify wrapped key is base64 encoded - _, err = ocrypto.Base64Decode([]byte(wrappedKey)) - require.NoError(t, err, "Wrapped key should be valid base64") - }) - - t.Run("wraps key with EC public key", func(t *testing.T) { - // Test elliptic curve key wrapping with ephemeral key generation - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - // Generate a real EC P-256 key pair for testing - ecKeyPair, err := ocrypto.NewECKeyPair(ocrypto.ECCModeSecp256r1) - require.NoError(t, err, "Should generate EC key pair") - - ecPublicKeyPEM, err := ecKeyPair.PublicKeyInPemFormat() - require.NoError(t, err, "Should get public key in PEM format") - - pubKeyInfo := keysplit.KASPublicKey{ - URL: testKAS1URL, - Algorithm: "ec:secp256r1", - KID: "test-kid", - PEM: ecPublicKeyPEM, - } - - wrappedKey, keyType, ephemeralPubKey, err := wrapKeyWithPublicKey(symKey, pubKeyInfo) - - require.NoError(t, err, "Should wrap key with EC public key") - assert.NotEmpty(t, wrappedKey, "Should return wrapped key") - assert.Equal(t, "eccWrapped", keyType, "EC keys should use 'eccWrapped' type") - assert.NotEmpty(t, ephemeralPubKey, "EC should generate ephemeral public key") - - // Verify ephemeral key is valid PEM - assert.True(t, strings.HasPrefix(ephemeralPubKey, "-----BEGIN PUBLIC KEY-----"), - "Ephemeral key should be in PEM format") - assert.True(t, strings.HasSuffix(ephemeralPubKey, "-----END PUBLIC KEY-----\n"), - "Ephemeral key should end with PEM footer") - }) - - t.Run("wraps key with X-Wing public key", func(t *testing.T) { - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - xwingKeyPair, err := ocrypto.NewXWingKeyPair() - require.NoError(t, err) - - xwingPublicKeyPEM, err := xwingKeyPair.PublicKeyInPemFormat() - require.NoError(t, err) - - pubKeyInfo := keysplit.KASPublicKey{ - URL: testKAS1URL, - Algorithm: string(ocrypto.HybridXWingKey), - KID: "test-kid", - PEM: xwingPublicKeyPEM, - } - - wrappedKey, keyType, ephemeralPubKey, err := wrapKeyWithPublicKey(symKey, pubKeyInfo) - - require.NoError(t, err, "Should wrap key with X-Wing public key") - assert.NotEmpty(t, wrappedKey) - assert.Equal(t, "hybrid-wrapped", keyType) - assert.Empty(t, ephemeralPubKey) - - decodedWrappedKey, err := ocrypto.Base64Decode([]byte(wrappedKey)) - require.NoError(t, err) - - privateKeyPEM, err := xwingKeyPair.PrivateKeyInPemFormat() - require.NoError(t, err) - dec, err := ocrypto.FromPrivatePEM(privateKeyPEM) - require.NoError(t, err) - - plaintext, err := dec.Decrypt(decodedWrappedKey) - require.NoError(t, err) - assert.Equal(t, symKey, plaintext) - }) - - t.Run("wraps key with P256+ML-KEM-768 public key", func(t *testing.T) { - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - keyPair, err := ocrypto.NewP256MLKEM768KeyPair() - require.NoError(t, err) - - publicKeyPEM, err := keyPair.PublicKeyInPemFormat() - require.NoError(t, err) - - pubKeyInfo := keysplit.KASPublicKey{ - URL: testKAS1URL, - Algorithm: string(ocrypto.HybridSecp256r1MLKEM768Key), - KID: "test-kid", - PEM: publicKeyPEM, - } - - wrappedKey, keyType, ephemeralPubKey, err := wrapKeyWithPublicKey(symKey, pubKeyInfo) - - require.NoError(t, err, "Should wrap key with P256+ML-KEM-768 public key") - assert.NotEmpty(t, wrappedKey) - assert.Equal(t, "hybrid-wrapped", keyType) - assert.Empty(t, ephemeralPubKey) - - decodedWrappedKey, err := ocrypto.Base64Decode([]byte(wrappedKey)) - require.NoError(t, err) - - privateKeyPEM, err := keyPair.PrivateKeyInPemFormat() - require.NoError(t, err) - dec, err := ocrypto.FromPrivatePEM(privateKeyPEM) - require.NoError(t, err) - - plaintext, err := dec.Decrypt(decodedWrappedKey) - require.NoError(t, err) - assert.Equal(t, symKey, plaintext) - }) - - t.Run("wraps key with P384+ML-KEM-1024 public key", func(t *testing.T) { - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - keyPair, err := ocrypto.NewP384MLKEM1024KeyPair() - require.NoError(t, err) - - publicKeyPEM, err := keyPair.PublicKeyInPemFormat() - require.NoError(t, err) - - pubKeyInfo := keysplit.KASPublicKey{ - URL: testKAS1URL, - Algorithm: string(ocrypto.HybridSecp384r1MLKEM1024Key), - KID: "test-kid", - PEM: publicKeyPEM, - } - - wrappedKey, keyType, ephemeralPubKey, err := wrapKeyWithPublicKey(symKey, pubKeyInfo) - - require.NoError(t, err, "Should wrap key with P384+ML-KEM-1024 public key") - assert.NotEmpty(t, wrappedKey) - assert.Equal(t, "hybrid-wrapped", keyType) - assert.Empty(t, ephemeralPubKey) - - decodedWrappedKey, err := ocrypto.Base64Decode([]byte(wrappedKey)) - require.NoError(t, err) - - privateKeyPEM, err := keyPair.PrivateKeyInPemFormat() - require.NoError(t, err) - dec, err := ocrypto.FromPrivatePEM(privateKeyPEM) - require.NoError(t, err) - - plaintext, err := dec.Decrypt(decodedWrappedKey) - require.NoError(t, err) - assert.Equal(t, symKey, plaintext) - }) - - t.Run("returns error for empty PEM", func(t *testing.T) { - // Test error handling for missing public key PEM - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - pubKeyInfo := keysplit.KASPublicKey{ - URL: testKAS1URL, - Algorithm: "rsa:2048", - KID: "test-kid", - PEM: "", // Empty PEM - } - - _, _, _, err = wrapKeyWithPublicKey(symKey, pubKeyInfo) - - require.Error(t, err, "Should return error for empty PEM") - assert.Contains(t, err.Error(), "public key PEM is empty", "Error should mention empty PEM") - }) - - t.Run("returns error for malformed PEM", func(t *testing.T) { - // Test error handling for invalid PEM format - symKey := make([]byte, 32) - _, err := rand.Read(symKey) - require.NoError(t, err) - - pubKeyInfo := keysplit.KASPublicKey{ - URL: testKAS1URL, - Algorithm: "rsa:2048", - KID: "test-kid", - PEM: "invalid-pem-data", - } - - _, _, _, err = wrapKeyWithPublicKey(symKey, pubKeyInfo) - - require.Error(t, err, "Should return error for malformed PEM") - }) -} - -func TestTdfSalt(t *testing.T) { - t.Run("generates consistent TDF salt", func(t *testing.T) { - // Test that tdfSalt() produces consistent output - salt1 := tdfSalt() - salt2 := tdfSalt() - - assert.Equal(t, salt1, salt2, "tdfSalt should produce consistent output") - assert.Len(t, salt1, 32, "Salt should be 32 bytes (SHA256 output)") - assert.NotEmpty(t, salt1, "Salt should not be empty") - }) -} diff --git a/sdk/experimental/tdf/keysplit_adapter.go b/sdk/experimental/tdf/keysplit_adapter.go new file mode 100644 index 0000000000..903b3150a8 --- /dev/null +++ b/sdk/experimental/tdf/keysplit_adapter.go @@ -0,0 +1,60 @@ +// Experimental: This package is EXPERIMENTAL and may change or be removed at any time + +package tdf + +import ( + "context" + + "github.com/opentdf/platform/protocol/go/policy" + "github.com/opentdf/platform/sdk" + "github.com/opentdf/platform/sdk/experimental/tdf/keysplit" +) + +// xorSplitter adapts this package's multi-KAS XOR splitter to the +// stable [sdk.KeySplitter] seam. +// +// This is why the experimental Writer is more than a rename of +// [sdk.NewChunkedWriter]: sdk.DefaultKeySplitter is single-KAS and +// ignores attributes, whereas [keysplit.XORSplitter] evaluates the full +// ABAC boolean expression and XOR-splits the DEK across every KAS the +// resulting clauses require. +// +// The two result shapes are field-identical; only the package differs. +// The one structural mismatch is where the default KAS enters: sdk +// passes it per Split call, keysplit takes it at construction, so the +// splitter is built inside Split rather than held on the adapter. +type xorSplitter struct{} + +// Split evaluates attrs and returns the DEK shares plus the wrapping +// key for each KAS they are addressed to. +// +// keysplit's errors (ErrNoDefaultKAS and friends) are returned +// unwrapped so callers can match on them as before. +func (xorSplitter) Split(ctx context.Context, attrs []*policy.Value, dek []byte, defaultKAS *policy.SimpleKasKey) (*sdk.SplitResult, error) { + splitter := keysplit.NewXORSplitter(keysplit.WithDefaultKAS(defaultKAS)) + res, err := splitter.GenerateSplits(ctx, attrs, dek) + if err != nil { + return nil, err + } + + out := &sdk.SplitResult{ + KASPublicKeys: make(map[string]sdk.KASPublicKey, len(res.KASPublicKeys)), + Splits: make([]sdk.Split, 0, len(res.Splits)), + } + for url, key := range res.KASPublicKeys { + out.KASPublicKeys[url] = sdk.KASPublicKey{ + Algorithm: key.Algorithm, + KID: key.KID, + PEM: key.PEM, + URL: key.URL, + } + } + for _, split := range res.Splits { + out.Splits = append(out.Splits, sdk.Split{ + Data: split.Data, + ID: split.ID, + KASURLs: split.KASURLs, + }) + } + return out, nil +} diff --git a/sdk/experimental/tdf/manifest.go b/sdk/experimental/tdf/manifest.go index e9268859eb..09a4cdd1fb 100644 --- a/sdk/experimental/tdf/manifest.go +++ b/sdk/experimental/tdf/manifest.go @@ -3,22 +3,9 @@ package tdf import ( - "encoding/hex" - "errors" - - "github.com/opentdf/platform/lib/ocrypto" "github.com/opentdf/platform/sdk" ) -// These are unchanged copies of what sdk defines unexported. They stay -// here only while this package still builds manifests itself; the -// delegation that removes their last callers is a follow-up. -const ( - kGMACPayloadLength = 16 - kSplitKeyType = "split" - kPolicyBindingAlg = "HS256" -) - // The manifest types below are aliases onto their // [github.com/opentdf/platform/sdk] counterparts, which own the definitions. // They are kept here so that existing importers of this experimental package @@ -102,24 +89,3 @@ type PolicyBody struct { DataAttributes []PolicyAttribute `json:"dataAttributes"` Dissem []string `json:"dissem"` } - -// calculateSignature is an unchanged copy of the sdk function of the -// same name, retained only until this package stops building manifests -// itself. -func calculateSignature(data []byte, secret []byte, alg IntegrityAlgorithm, isLegacyTDF bool) (string, error) { - if alg == HS256 { - hmac := ocrypto.CalculateSHA256Hmac(secret, data) - if isLegacyTDF { - return hex.EncodeToString(hmac), nil - } - return string(hmac), nil - } - if kGMACPayloadLength > len(data) { - return "", errors.New("fail to create gmac signature") - } - - if isLegacyTDF { - return hex.EncodeToString(data[len(data)-kGMACPayloadLength:]), nil - } - return string(data[len(data)-kGMACPayloadLength:]), nil -} diff --git a/sdk/experimental/tdf/options.go b/sdk/experimental/tdf/options.go index e2c7e70d31..15b71ac796 100644 --- a/sdk/experimental/tdf/options.go +++ b/sdk/experimental/tdf/options.go @@ -11,6 +11,12 @@ import "github.com/opentdf/platform/protocol/go/policy" // - GMAC: Galois Message Authentication Code, faster but requires AES-GCM support // // The algorithm choice affects both segment-level and root-level integrity verification. +// +// Unlike the manifest and assertion types in this package, this is not an +// alias onto [sdk.IntegrityAlgorithm]: that one is itself an alias for int, so +// no methods can be attached to it, and aliasing would silently drop String() +// from this package's public API. The underlying values match, so the two +// convert freely. type IntegrityAlgorithm int // String returns the string representation of the integrity algorithm. @@ -60,6 +66,10 @@ type WriterConfig struct { // initialDefaultKAS allows callers to provide a default KAS at writer creation time. // This will be used during Finalize() if no default KAS is provided there. initialDefaultKAS *policy.SimpleKasKey + + // targetMode is the TDF spec version to write for, as semver. + // Empty selects the current format. See WithTargetMode. + targetMode string } // ReaderConfig contains configuration options for TDF Reader creation. @@ -78,7 +88,7 @@ type ReaderConfig struct { // Example usage: // // writer, err := NewWriter(ctx, WithIntegrityAlgorithm(GMAC)) -// finalBytes, manifest, err := writer.Finalize(ctx, WithPayloadMimeType("text/plain")) +// result, err := writer.Finalize(ctx, WithPayloadMimeType("text/plain")) type Option[T any] func(T) // WithIntegrityAlgorithm sets the algorithm for root integrity signature calculation. @@ -180,9 +190,9 @@ type WriterFinalizeConfig struct { // Used by readers to determine appropriate content handling. payloadMimeType string - // keepSegments indicates caller-provided segment indices to keep when finalizing. - // Indices must form a contiguous prefix [0..K]. If empty, all written - // segments (default behavior) are used. + // keepSegments names the segments the manifest should describe, + // ascending and possibly sparse. If empty, all written segments + // (default behavior) are used. keepSegments []int } @@ -198,7 +208,7 @@ type WriterFinalizeConfig struct { // // Example: // -// finalBytes, manifest, err := writer.Finalize(ctx, +// result, err := writer.Finalize(ctx, // WithEncryptedMetadata("classification: secret"), // ) func WithEncryptedMetadata(metadata string) Option[*WriterFinalizeConfig] { @@ -218,7 +228,7 @@ func WithEncryptedMetadata(metadata string) Option[*WriterFinalizeConfig] { // // Example: // -// finalBytes, manifest, err := writer.Finalize(ctx, +// result, err := writer.Finalize(ctx, // WithPayloadMimeType("application/json"), // ) func WithPayloadMimeType(mimeType string) Option[*WriterFinalizeConfig] { @@ -227,10 +237,17 @@ func WithPayloadMimeType(mimeType string) Option[*WriterFinalizeConfig] { } } -// WithSegments restricts finalization to the provided segment indices and order. -// The order provided is used as the logical payload order. Indices may be sparse -// but must refer to segments that were written. When omitted, all present indices -// are used in ascending order. +// WithSegments names the segments the finalized manifest describes. +// When omitted, all written segments are used in ascending index order. +// +// Indices may be sparse -- a caller that reserves a fixed block of +// indices per upload part and fills only the front of each block writes +// gaps by construction -- but they must be a prefix of the written +// segments in ascending index order. They may drop from the end; they +// may not reorder or skip. The payload is laid out in sorted index +// order, so a manifest that disagrees would not describe the bytes on +// disk. For the same reason the caller must concatenate each segment's +// TDFData in ascending index order. func WithSegments(indices []int) Option[*WriterFinalizeConfig] { return func(c *WriterFinalizeConfig) { c.keepSegments = indices @@ -257,7 +274,7 @@ func WithSegments(indices []int) Option[*WriterFinalizeConfig] { // Pem: kasPublicKeyPEM, // }, // } -// finalBytes, manifest, err := writer.Finalize(ctx, WithDefaultKAS(kasKey)) +// result, err := writer.Finalize(ctx, WithDefaultKAS(kasKey)) func WithDefaultKAS(kas *policy.SimpleKasKey) Option[*WriterFinalizeConfig] { return func(c *WriterFinalizeConfig) { c.defaultKas = kas @@ -287,34 +304,49 @@ func WithDefaultKAS(kas *policy.SimpleKasKey) Option[*WriterFinalizeConfig] { // Grants: []*policy.KeyAccessServer{kasConfig}, // }, // } -// finalBytes, manifest, err := writer.Finalize(ctx, WithAttributeValues(attributes)) +// result, err := writer.Finalize(ctx, WithAttributeValues(attributes)) func WithAttributeValues(values []*policy.Value) Option[*WriterFinalizeConfig] { return func(c *WriterFinalizeConfig) { c.attributes = values } } -// WithExcludeVersionFromManifest controls version information in the manifest. +// WithExcludeVersionFromManifest is a no-op and always has been: the +// manifest builder never read the flag it sets, so schemaVersion is +// emitted either way. // -// When set to true, excludes TDF specification version information from -// the manifest. This may be needed for compatibility with older TDF readers -// that don't expect version fields. +// Omitting schemaVersion is not independently useful in any case. A +// reader treats a missing schemaVersion as "predates 4.3.0" and then +// expects hex-then-base64 signatures, which are decided per segment at +// write time -- long before Finalize sees this option. The two must be +// set together, which is what [WithTargetMode] does. // -// Generally should be left as default (false) unless specific compatibility -// requirements exist. -// -// Example: -// -// // For compatibility with legacy readers -// finalBytes, manifest, err := writer.Finalize(ctx, -// WithExcludeVersionFromManifest(true), -// ) +// Deprecated: use [WithTargetMode] at writer construction. func WithExcludeVersionFromManifest(exclude bool) Option[*WriterFinalizeConfig] { return func(c *WriterFinalizeConfig) { c.excludeVersionFromManifest = exclude } } +// WithTargetMode targets a specific TDF spec version, given as a semver +// string such as "4.2.2". +// +// Below 4.3.0 the writer emits the legacy wire format: segment, root, +// and assertion signatures are hex-encoded before base64, and +// schemaVersion is omitted from the manifest, which is how those +// readers detect it. The two travel together -- a manifest carrying one +// without the other cannot be verified by any reader. +// +// An empty mode selects the current format. +// +// A malformed semver string is reported by NewWriter, not here: this +// package's Option signature has no error return. +func WithTargetMode(mode string) Option[*WriterConfig] { + return func(c *WriterConfig) { + c.targetMode = mode + } +} + // WithAssertions includes cryptographic assertions in the TDF. // // Assertions provide additional integrity verification and can include @@ -343,7 +375,7 @@ func WithExcludeVersionFromManifest(exclude bool) Option[*WriterFinalizeConfig] // Value: `{"retention_days": 90}`, // }, // } -// finalBytes, manifest, err := writer.Finalize(ctx, WithAssertions(assertion)) +// result, err := writer.Finalize(ctx, WithAssertions(assertion)) func WithAssertions(assertions ...AssertionConfig) Option[*WriterFinalizeConfig] { return func(c *WriterFinalizeConfig) { c.assertions = assertions diff --git a/sdk/experimental/tdf/writer.go b/sdk/experimental/tdf/writer.go index 2a58af6c87..aefa4fa9e4 100644 --- a/sdk/experimental/tdf/writer.go +++ b/sdk/experimental/tdf/writer.go @@ -3,34 +3,13 @@ package tdf import ( - "bytes" "context" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "hash/crc32" "io" "log/slog" - "sort" - "sync" + "sync/atomic" - "github.com/google/uuid" - "github.com/opentdf/platform/lib/ocrypto" "github.com/opentdf/platform/protocol/go/policy" - "github.com/opentdf/platform/sdk/experimental/tdf/keysplit" - "github.com/opentdf/platform/sdk/internal/zipstream" -) - -const ( - // kKeySize is the AES key size in bytes (256-bit key) - kKeySize = 32 - // kGCMCipherAlgorithm specifies the encryption algorithm used for TDF payloads - kGCMCipherAlgorithm = "AES-256-GCM" - // tdfAsZip indicates the TDF uses ZIP as the container format - tdfAsZip = "zip" - // tdfZipReference indicates the payload is stored as a reference in the ZIP - tdfZipReference = "reference" + "github.com/opentdf/platform/sdk" ) // SegmentResult contains the result of writing a segment @@ -51,13 +30,18 @@ type FinalizeResult struct { EncryptedSize int64 `json:"encryptedSize"` // Total encrypted size } +// These are the stable SDK's error values rather than copies of them, +// so errors.Is matches whether a caller compares against the +// experimental or the sdk-scoped name. var ( // ErrAlreadyFinalized is returned when attempting operations on a finalized writer - ErrAlreadyFinalized = errors.New("tdf is already finalized") + ErrAlreadyFinalized = sdk.ErrChunkedAlreadyFinalized // ErrInvalidSegmentIndex is returned for negative segment indices - ErrInvalidSegmentIndex = errors.New("invalid segment index") + ErrInvalidSegmentIndex = sdk.ErrChunkedInvalidSegmentIndex + // ErrMissingSegmentZero is returned when Finalize is called without segment 0 + ErrMissingSegmentZero = sdk.ErrChunkedMissingSegmentZero // ErrSegmentAlreadyWritten is returned when trying to write to an existing segment index - ErrSegmentAlreadyWritten = errors.New("segment already written") + ErrSegmentAlreadyWritten = sdk.ErrChunkedSegmentAlreadyWritten ) // Writer provides streaming TDF creation with out-of-order segment support. @@ -67,15 +51,19 @@ var ( // and proper ZIP archive structure generation. // // Key features: -// - Variable-length segments with sparse index support +// - Variable-length segments with sparse index support above index 0 // - Out-of-order segment writing without buffering payloads // - Memory-efficient handling through segment cleanup // - Cryptographic assertions and integrity verification // - Custom attribute-based access controls // -// Thread safety: Writers require external synchronization for concurrent access. -// Each WriteSegment call must be serialized, but multiple Writers can operate -// independently. +// Thread safety: WriteSegment may be called concurrently for distinct +// indices, but not twice for the same index. +// +// The writing itself is [sdk.ChunkedWriter]; this type adds the +// multi-KAS ABAC key splitting in [xorSplitter] and this package's +// option style. Callers that only need a single KAS can use +// [sdk.NewChunkedWriter] directly. // // Example usage: // @@ -83,44 +71,23 @@ var ( // if err != nil { // return err // } -// defer writer.Close() // // // Write segments (can be out-of-order) // _, err = writer.WriteSegment(ctx, 1, []byte("second")) // _, err = writer.WriteSegment(ctx, 0, []byte("first")) // // // Finalize with attributes -// finalBytes, manifest, err := writer.Finalize(ctx, WithAttributeValues(attrs)) +// result, err := writer.Finalize(ctx, WithAttributeValues(attrs)) type Writer struct { // WriterConfig embeds configuration options for the TDF writer WriterConfig - // archiveWriter handles the underlying ZIP archive creation - archiveWriter zipstream.SegmentWriter - - // State management - mutex sync.RWMutex // Protects concurrent access to writer state - finalized bool // Whether Finalize() has been called - - // manifest holds the finalized manifest after Finalize() is called. - // Before finalization, GetManifest() will synthesize a stub manifest - // from the current writer state. Do not rely on the stub for - // verification — it is informational only until Finalize completes. - manifest *Manifest + // inner is the stable per-segment writer this type delegates to. + inner sdk.ChunkedWriter - // segments stores segment metadata using sparse map for memory efficiency - // Maps segment index to Segment metadata (hash, size information) - segments map[int]*Segment - // maxSegmentIndex tracks the highest segment index written - maxSegmentIndex int - - // Cryptographic state - dek []byte // Data Encryption Key (32-byte AES key) - block ocrypto.AesGcm // AES-GCM cipher for segment encryption - - // Initial settings provided at Writer creation; used by Finalize if not overridden - initialAttributes []*policy.Value - initialDefaultKAS *policy.SimpleKasKey + // finalized mirrors inner's state so GetManifest can warn that a + // pre-finalize manifest is a stub. inner does not expose it. + finalized atomic.Bool } // NewWriter creates a new experimental TDF Writer with streaming support. @@ -128,15 +95,13 @@ type Writer struct { // The writer is initialized with secure defaults: // - HS256 integrity algorithms for both root and segment verification // - AES-256-GCM encryption for all segments -// - Dynamic segment expansion supporting sparse indices +// - Dynamic segment expansion supporting sparse indices (index 0 always required) // - Memory-efficient segment processing // // Configuration options can be provided to customize: // - Integrity algorithm selection (HS256, GMAC) // - Segment integrity algorithm (independent of root algorithm) -// -// The writer generates a unique Data Encryption Key (DEK) and initializes -// the underlying archive writer for ZIP structure management. +// - Attributes and default KAS to fall back on at Finalize time // // Returns an error if: // - DEK generation fails (cryptographic entropy issues) @@ -153,41 +118,28 @@ type Writer struct { // WithIntegrityAlgorithm(GMAC), // WithSegmentIntegrityAlgorithm(HS256), // ) -func NewWriter(_ context.Context, opts ...Option[*WriterConfig]) (*Writer, error) { - // Initialize Config +func NewWriter(ctx context.Context, opts ...Option[*WriterConfig]) (*Writer, error) { config := &WriterConfig{ integrityAlgorithm: HS256, segmentIntegrityAlgorithm: HS256, } - for _, opt := range opts { opt(config) } - // Initialize archive writer - start with 1 segment and expand dynamically - archiveWriter := zipstream.NewSegmentTDFWriter(1, zipstream.WithZip64()) - - // Generate DEK - dek, err := ocrypto.RandomBytes(kKeySize) + inner, err := sdk.NewChunkedWriter(ctx, + sdk.WithChunkedIntegrityAlgorithm(sdk.IntegrityAlgorithm(config.integrityAlgorithm)), + sdk.WithChunkedSegmentIntegrityAlgorithm(sdk.IntegrityAlgorithm(config.segmentIntegrityAlgorithm)), + sdk.WithChunkedInitialAttributes(config.initialAttributes), + sdk.WithChunkedDefaultKAS(config.initialDefaultKAS), + sdk.WithChunkedKeySplitter(xorSplitter{}), + sdk.WithChunkedTargetMode(config.targetMode), + ) if err != nil { return nil, err } - // Initialize AES GCM Provider - block, err := ocrypto.NewAESGcm(dek) - if err != nil { - return nil, err - } - - return &Writer{ - WriterConfig: *config, - archiveWriter: archiveWriter, - dek: dek, - segments: make(map[int]*Segment), // Initialize sparse storage - block: block, - initialAttributes: config.initialAttributes, - initialDefaultKAS: config.initialDefaultKAS, - }, nil + return &Writer{WriterConfig: *config, inner: inner}, nil } // WriteSegment encrypts and writes a data segment at the specified index. @@ -198,26 +150,18 @@ func NewWriter(_ context.Context, opts ...Option[*WriterConfig]) (*Writer, error // // Parameters: // - ctx: Context for cancellation and timeout control -// - index: Zero-based segment index (must be non-negative, sparse indices supported) +// - index: Zero-based segment index (must be non-negative; sparse indices supported, +// but index 0 must eventually be written) // - data: Raw data to encrypt and store in this segment // // Returns the encrypted segment bytes that should be stored/uploaded, and any error. -// The returned bytes include ZIP structure elements and can be assembled in any order. -// -// The function performs: -// 1. Input validation (index >= 0, writer not finalized, no duplicate segments) -// 2. AES-256-GCM encryption of the segment data -// 3. HMAC signature calculation for integrity verification -// 4. ZIP archive segment creation through the archive layer -// -// Memory optimization: Uses sparse storage to avoid O(n²) memory growth -// for high or non-contiguous segment indices. +// The returned bytes include ZIP structure elements. They must be concatenated in +// ascending index order to form the payload, whatever order they were produced in. // // Error conditions: // - ErrAlreadyFinalized: Writer has been finalized, no more segments accepted // - ErrInvalidSegmentIndex: Negative index provided // - ErrSegmentAlreadyWritten: Segment index already contains data -// - Context cancellation: If ctx.Done() is signaled // - Encryption errors: AES-GCM operation failures // - Archive errors: ZIP structure creation failures // @@ -231,97 +175,25 @@ func NewWriter(_ context.Context, opts ...Option[*WriterConfig]) (*Writer, error // uploadToS3(segment0, "part-000") // uploadToS3(segment1, "part-001") func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*SegmentResult, error) { - w.mutex.Lock() - - if w.finalized { - w.mutex.Unlock() - return nil, ErrAlreadyFinalized - } - - if index < 0 { - w.mutex.Unlock() - return nil, ErrInvalidSegmentIndex - } - - // Check for duplicate segments using map lookup - if _, exists := w.segments[index]; exists { - w.mutex.Unlock() - return nil, ErrSegmentAlreadyWritten - } - - if index > w.maxSegmentIndex { - w.maxSegmentIndex = index - } - seg := &Segment{ - Size: -1, // indicates not filled yet - } - w.segments[index] = seg - - w.mutex.Unlock() - - // Encrypt directly without unnecessary copying - the archive layer will handle copying if needed - segmentCipher, nonce, err := w.block.EncryptInPlace(data) + res, err := w.inner.WriteSegment(ctx, index, data) if err != nil { return nil, err } - segmentSig, err := calculateSignature(segmentCipher, w.dek, w.segmentIntegrityAlgorithm, false) // Don't ever hex encode new tdf's - if err != nil { - return nil, err - } - - segmentHash := string(ocrypto.Base64Encode([]byte(segmentSig))) - w.mutex.Lock() - seg.Size = int64(len(data)) - seg.EncryptedSize = int64(len(segmentCipher)) + int64(len(nonce)) - seg.Hash = segmentHash - w.mutex.Unlock() - - crc := crc32.NewIEEE() - _, err = crc.Write(nonce) - if err != nil { - return nil, err - } - _, err = crc.Write(segmentCipher) - if err != nil { - return nil, err - } - header, err := w.archiveWriter.WriteSegment(ctx, index, uint64(seg.EncryptedSize), crc.Sum32()) - if err != nil { - return nil, err - } - var reader io.Reader - if len(header) == 0 { - reader = io.MultiReader(bytes.NewReader(nonce), bytes.NewReader(segmentCipher)) - } else { - reader = io.MultiReader(bytes.NewReader(header), bytes.NewReader(nonce), bytes.NewReader(segmentCipher)) - } - return &SegmentResult{ - TDFData: reader, - Index: index, - Hash: seg.Hash, - PlaintextSize: seg.Size, - EncryptedSize: seg.EncryptedSize, + TDFData: res.TDFData, + Index: res.Index, + Hash: res.Hash, + PlaintextSize: res.PlaintextSize, + EncryptedSize: res.EncryptedSize, }, nil } // Finalize completes TDF creation and returns the final bytes and manifest. // -// This method must be called after all segments have been written. It performs: -// 1. Validates all segments are present (no missing indices from 0 to maxSegmentIndex) -// 2. Generates cryptographic splits for key access controls -// 3. Builds the TDF policy from provided attributes -// 4. Creates cryptographic assertions if specified -// 5. Calculates root integrity signature over all segment hashes -// 6. Generates the complete TDF manifest -// 7. Finalizes the ZIP archive structure -// -// The finalization process handles: -// - Key splitting for attribute-based access controls -// - Policy generation from attribute values -// - Encrypted metadata storage in key access objects -// - Manifest JSON generation and validation -// - ZIP central directory and data descriptor creation +// This method must be called after all segments have been written. It +// generates the key splits for attribute-based access control, builds the +// policy, signs any assertions, calculates the root integrity signature over +// all segment hashes, and closes the ZIP archive. // // Parameters: // - ctx: Context for cancellation and timeout control @@ -333,30 +205,24 @@ func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*Seg // - WithPayloadMimeType: Specify payload MIME type // - WithAssertions: Add cryptographic assertions // - WithDefaultKAS: Set default Key Access Server -// -// Returns: -// - finalBytes: Complete ZIP archive bytes ready for storage/transmission -// - manifest: TDF manifest containing encryption and integrity information -// - error: Any error during finalization process +// - WithSegments: Choose which written segments the manifest describes // // Error conditions: // - ErrAlreadyFinalized: Finalize already called -// - Missing segment 0: Index 0 carries the payload's ZIP local file header, which -// every recorded offset is measured from, so a write set that omits it is -// rejected. Gaps between the remaining indices are legal (e.g., segments 0,1,3 -// with 2 missing); order is inferred by sorting whichever indices are present. +// - ErrMissingSegmentZero: segment 0 was never written; it carries the payload's +// ZIP local file header, which every recorded offset is measured from +// - Missing segments: an index named by WithSegments that was never written // - Key splitting failures: Invalid attributes or KAS configuration // - Manifest generation errors: JSON marshaling failures // - Archive finalization errors: ZIP structure generation failures -// - Context cancellation: If ctx.Done() is signaled // // Example: // // // Basic finalization -// finalBytes, manifest, err := writer.Finalize(ctx) +// result, err := writer.Finalize(ctx) // // // With attributes and metadata -// finalBytes, manifest, err := writer.Finalize(ctx, +// result, err := writer.Finalize(ctx, // WithAttributeValues(attrs), // WithEncryptedMetadata("sensitive info"), // WithPayloadMimeType("application/json"), @@ -364,50 +230,18 @@ func (w *Writer) WriteSegment(ctx context.Context, index int, data []byte) (*Seg // // Performance note: Finalization is O(n) where n is the number of segments. // Memory usage is proportional to manifest size, not total data size. - func (w *Writer) Finalize(ctx context.Context, opts ...Option[*WriterFinalizeConfig]) (*FinalizeResult, error) { - w.mutex.Lock() - defer w.mutex.Unlock() - - if w.finalized { - return nil, ErrAlreadyFinalized - } - - cfg := &WriterFinalizeConfig{ - attributes: make([]*policy.Value, 0), - encryptedMetadata: "", - payloadMimeType: "application/octet-stream", - } - for _, opt := range opts { - opt(cfg) - } - manifest, totalPlaintextSize, totalEncryptedSize, err := w.getManifest(ctx, cfg) - if err != nil { - return nil, err - } - manifestBytes, err := json.Marshal(manifest) - if err != nil { - return nil, err - } - - finalBytes, err := w.archiveWriter.Finalize(ctx, manifestBytes) + res, err := w.inner.Finalize(ctx, finalizeOptions(opts)...) if err != nil { return nil, err } - - if err := w.archiveWriter.Close(); err != nil { - return nil, err - } - - // Persist the final manifest for later retrieval via GetManifest. - w.manifest = manifest - w.finalized = true + w.finalized.Store(true) return &FinalizeResult{ - Data: finalBytes, - Manifest: manifest, - TotalSegments: len(manifest.Segments), - TotalSize: totalPlaintextSize, - EncryptedSize: totalEncryptedSize, + Data: res.Data, + Manifest: res.Manifest, + TotalSegments: res.TotalSegments, + TotalSize: res.TotalSize, + EncryptedSize: res.EncryptedSize, }, nil } @@ -419,13 +253,25 @@ func (w *Writer) Finalize(ctx context.Context, opts ...Option[*WriterFinalizeCon // from the writer's current state (segments present so far, algorithm // selections, and payload defaults). This pre-finalize manifest is not // complete and must not be used for verification; it is provided for -// informational or client-side pre-calculation purposes only. -// -// No logging is performed; callers should consult this documentation for -// the caveat about pre-finalize state. +// informational or client-side pre-calculation purposes only. A +// warning is logged in that case. func (w *Writer) GetManifest(ctx context.Context, opts ...Option[*WriterFinalizeConfig]) (*Manifest, error) { - w.mutex.RLock() - defer w.mutex.RUnlock() + if !w.finalized.Load() { + slog.Warn("getmanifest called before finalize; returned manifest is a stub and not complete, pre-finalize state may not include all segments or attributes.") + } + return w.inner.GetManifest(ctx, finalizeOptions(opts)...) +} + +// finalizeOptions translates this package's finalize options into the +// stable writer's equivalents. +// +// The two option sets are applied in sequence rather than mapped +// one-to-one: this package's Option is a plain mutator over a config +// struct, so the config is materialized first and then read off. That +// also preserves the defaults callers have always seen -- notably the +// "application/octet-stream" MIME type -- independent of whichever +// defaults the stable writer happens to use. +func finalizeOptions(opts []Option[*WriterFinalizeConfig]) []sdk.ChunkedFinalizeOption { cfg := &WriterFinalizeConfig{ attributes: make([]*policy.Value, 0), encryptedMetadata: "", @@ -434,250 +280,12 @@ func (w *Writer) GetManifest(ctx context.Context, opts ...Option[*WriterFinalize for _, opt := range opts { opt(cfg) } - if !w.finalized { - slog.Warn("getmanifest called before finalize; returned manifest is a stub and not complete, pre-finalize state may not include all segments or attributes.") - } - - manifest, _, _, err := w.getManifest(ctx, cfg) - if err != nil { - return nil, err - } - return manifest, nil -} - -func (w *Writer) getManifest(ctx context.Context, cfg *WriterFinalizeConfig) (*Manifest, int64, int64, error) { - // If already finalized and we have the final manifest, return a copy. - if w.finalized && w.manifest != nil { - return cloneManifest(w.manifest), 0, 0, nil - } - // Archive layer will infer the same order by sorting present indices. - // Merge writer-level initial settings if finalize options omitted them. - if len(cfg.attributes) == 0 && len(w.initialAttributes) > 0 { - cfg.attributes = w.initialAttributes - } - if cfg.defaultKas == nil && w.initialDefaultKAS != nil { - cfg.defaultKas = w.initialDefaultKAS - } - - manifest := &Manifest{ - TDFVersion: TDFSpecVersion, - Payload: Payload{ - MimeType: cfg.payloadMimeType, - Protocol: tdfAsZip, - Type: tdfZipReference, - URL: zipstream.TDFPayloadFileName, - IsEncrypted: true, - }, - } - // Determine finalize order by collecting all present segment indices and sorting. - // This densifies sparse indices automatically and ignores any gaps. - order := make([]int, 0, len(w.segments)) - for idx := range w.segments { - order = append(order, idx) - } - sort.Ints(order) - // If caller provided keepSegments, restrict to that subset and order. - if len(cfg.keepSegments) > 0 { - subset := make([]int, 0, len(cfg.keepSegments)) - seen := make(map[int]struct{}, len(cfg.keepSegments)) - for _, idx := range cfg.keepSegments { - if idx < 0 { - return nil, 0, 0, fmt.Errorf("WithSegments contains invalid index %d (must be >= 0)", idx) - } - if _, ok := w.segments[idx]; !ok { - return nil, 0, 0, fmt.Errorf("WithSegments references segment %d which was not written", idx) - } - if _, dup := seen[idx]; dup { - return nil, 0, 0, fmt.Errorf("WithSegments contains duplicate index %d", idx) - } - seen[idx] = struct{}{} - subset = append(subset, idx) - } - order = subset - } - - // Generate splits using the splitter - splitter := keysplit.NewXORSplitter(keysplit.WithDefaultKAS(cfg.defaultKas)) - result, err := splitter.GenerateSplits(ctx, cfg.attributes, w.dek) - if err != nil { - return nil, 0, 0, err - } - - // Build key access objects from the splits - policyBytes, err := buildPolicy(cfg.attributes) - if err != nil { - return nil, 0, 0, err - } - - encryptInfo := EncryptionInformation{ - KeyAccessType: kSplitKeyType, - Policy: string(ocrypto.Base64Encode(policyBytes)), - Method: Method{ - Algorithm: kGCMCipherAlgorithm, - IsStreamable: true, - }, - IntegrityInformation: IntegrityInformation{ - // Copy segments to manifest for integrity verification in finalize order - Segments: make([]Segment, len(order)), - RootSignature: RootSignature{}, - }, - } - - // Copy segments to manifest in finalize order (pack densely) - for i, idx := range order { - if segment, exists := w.segments[idx]; exists { - encryptInfo.Segments[i] = *segment - } - } - - // Set default segment sizes for reader compatibility - // Use the first segment as the default (streaming TDFs have variable segment sizes) - if firstSegment, exists := w.segments[0]; exists { - encryptInfo.DefaultSegmentSize = firstSegment.Size - encryptInfo.DefaultEncryptedSegSize = firstSegment.EncryptedSize - } - - // Set segment hash algorithm - encryptInfo.SegmentHashAlgorithm = w.segmentIntegrityAlgorithm.String() - - var aggregateHash bytes.Buffer - // Calculate totals and iterate through segments in finalize order - var totalPlaintextSize, totalEncryptedSize int64 - for _, i := range order { - segment, exists := w.segments[i] - // if size is negative, segment was not written, finalized has been called too early - if !exists || w.segments[i].Size < 0 { - return nil, 0, 0, fmt.Errorf("segment %d not written; cannot finalize", i) - } - if segment.Hash != "" { - // Accumulate sizes for result - totalPlaintextSize += segment.Size - totalEncryptedSize += segment.EncryptedSize - - // Decode the base64-encoded segment hash to match reader validation - decodedHash, err := ocrypto.Base64Decode([]byte(segment.Hash)) - if err != nil { - return nil, 0, 0, fmt.Errorf("failed to decode segment hash: %w", err) - } - aggregateHash.Write(decodedHash) - continue - } - return nil, 0, 0, errors.New("empty segment hash") - } - - rootSignature, err := calculateSignature(aggregateHash.Bytes(), w.dek, w.integrityAlgorithm, false) - if err != nil { - return nil, 0, 0, err - } - encryptInfo.RootSignature = RootSignature{ - Algorithm: w.integrityAlgorithm.String(), - Signature: string(ocrypto.Base64Encode([]byte(rootSignature))), - } - - keyAccessList, err := buildKeyAccessObjects(result, policyBytes, cfg.encryptedMetadata) - if err != nil { - return nil, 0, 0, err - } - - encryptInfo.KeyAccessObjs = keyAccessList - manifest.EncryptionInformation = encryptInfo - - signedAssertions, err := w.buildAssertions(aggregateHash.Bytes(), cfg.assertions) - if err != nil { - return nil, 0, 0, err - } - - manifest.Assertions = signedAssertions - return manifest, totalPlaintextSize, totalEncryptedSize, nil -} - -// cloneManifest makes a shallow-deep copy of a Manifest to avoid callers -// mutating internal writer state. -func cloneManifest(in *Manifest) *Manifest { - if in == nil { - return nil - } - out := *in // copy by value - - // Copy slices to new backing arrays - if in.KeyAccessObjs != nil { - out.KeyAccessObjs = append([]KeyAccess(nil), in.KeyAccessObjs...) - } - if in.Segments != nil { - out.Segments = append([]Segment(nil), in.Segments...) - } - if in.Assertions != nil { - out.Assertions = append([]Assertion(nil), in.Assertions...) - } - return &out -} - -func buildPolicy(values []*policy.Value) ([]byte, error) { - policy := &Policy{ - UUID: uuid.NewString(), - Body: PolicyBody{ - DataAttributes: make([]PolicyAttribute, 0), - Dissem: make([]string, 0), - }, - } - - for _, value := range values { - policy.Body.DataAttributes = append(policy.Body.DataAttributes, PolicyAttribute{ - Attribute: value.GetFqn(), - }) - } - policyBytes, err := json.Marshal(policy) - if err != nil { - return nil, err - } - - return policyBytes, nil -} - -func (w *Writer) buildAssertions(aggregateHash []byte, assertions []AssertionConfig) ([]Assertion, error) { - signedAssertion := make([]Assertion, 0) - for _, assertion := range assertions { - // Store a temporary assertion - tmpAssertion := Assertion{} - - tmpAssertion.ID = assertion.ID - tmpAssertion.Type = assertion.Type - tmpAssertion.Scope = assertion.Scope - tmpAssertion.Statement = assertion.Statement - tmpAssertion.AppliesToState = assertion.AppliesToState - - hashOfAssertionAsHex, err := tmpAssertion.GetHash() - if err != nil { - return nil, err - } - - hashOfAssertion := make([]byte, hex.DecodedLen(len(hashOfAssertionAsHex))) - _, err = hex.Decode(hashOfAssertion, hashOfAssertionAsHex) - if err != nil { - return nil, fmt.Errorf("error decoding hex string: %w", err) - } - - var completeHashBuilder bytes.Buffer - completeHashBuilder.Write(aggregateHash) - completeHashBuilder.Write(hashOfAssertion) - - encoded := ocrypto.Base64Encode(completeHashBuilder.Bytes()) - - assertionSigningKey := AssertionKey{} - - // Set default to HS256 and payload key - assertionSigningKey.Alg = AssertionKeyAlgHS256 - assertionSigningKey.Key = w.dek - - if !assertion.SigningKey.IsEmpty() { - assertionSigningKey = assertion.SigningKey - } - - if err := tmpAssertion.Sign(string(hashOfAssertionAsHex), string(encoded), assertionSigningKey); err != nil { - return nil, fmt.Errorf("failed to sign assertion: %w", err) - } - - signedAssertion = append(signedAssertion, tmpAssertion) + return []sdk.ChunkedFinalizeOption{ + sdk.WithChunkedAttributes(cfg.attributes), + sdk.WithChunkedDefaultKASForFinalize(cfg.defaultKas), + sdk.WithChunkedEncryptedMetadata(cfg.encryptedMetadata), + sdk.WithChunkedMimeType(cfg.payloadMimeType), + sdk.WithChunkedSegments(cfg.keepSegments), + sdk.WithChunkedAssertions(cfg.assertions), } - return signedAssertion, nil } diff --git a/sdk/experimental/tdf/writer_test.go b/sdk/experimental/tdf/writer_test.go index 300b5746be..ba662a5696 100644 --- a/sdk/experimental/tdf/writer_test.go +++ b/sdk/experimental/tdf/writer_test.go @@ -5,6 +5,7 @@ package tdf import ( "bytes" "crypto/rand" + "encoding/hex" "encoding/json" "os" "path/filepath" @@ -270,21 +271,20 @@ func testBasicTDFCreationFlow(t *testing.T) { assert.NotNil(t, writer, "Writer should not be nil") // Verify initial state - assert.False(t, writer.finalized, "Writer should not be finalized initially") - assert.Empty(t, writer.segments, "Segments should be empty initially") - assert.Len(t, writer.dek, 32, "DEK should be 32 bytes") + assert.False(t, writer.finalized.Load(), "Writer should not be finalized initially") // Write a single segment testData := []byte("Hello, TDF World!") - zipBytes, err := writer.WriteSegment(ctx, 0, testData) + segResult, err := writer.WriteSegment(ctx, 0, testData) require.NoError(t, err, "Failed to write segment") - assert.NotEmpty(t, zipBytes, "Zip bytes should not be empty") + require.NotNil(t, segResult, "Segment result should not be nil") + assert.NotNil(t, segResult.TDFData, "Zip bytes should not be empty") // Verify segment was recorded - assert.Len(t, writer.segments, 1, "Should have one segment") - assert.Equal(t, int64(len(testData)), writer.segments[0].Size, "Segment size should match input data") - assert.NotEmpty(t, writer.segments[0].Hash, "Segment hash should be set") - assert.Greater(t, writer.segments[0].EncryptedSize, writer.segments[0].Size, "Encrypted size should be larger due to GCM overhead") + assert.Equal(t, 0, segResult.Index, "Segment index should match") + assert.Equal(t, int64(len(testData)), segResult.PlaintextSize, "Segment size should match input data") + assert.NotEmpty(t, segResult.Hash, "Segment hash should be set") + assert.Greater(t, segResult.EncryptedSize, segResult.PlaintextSize, "Encrypted size should be larger due to GCM overhead") // Finalize with attributes that have proper KAS setup attributes := []*policy.Value{ @@ -299,7 +299,7 @@ func testBasicTDFCreationFlow(t *testing.T) { validateManifestSchema(t, finalizeResult.Manifest) // Verify finalized state - assert.True(t, writer.finalized, "Writer should be finalized") + assert.True(t, writer.finalized.Load(), "Writer should be finalized") // Verify manifest structure assert.Equal(t, TDFSpecVersion, finalizeResult.Manifest.TDFVersion, "TDF version should match expected") @@ -316,7 +316,7 @@ func testBasicTDFCreationFlow(t *testing.T) { assert.NotEmpty(t, keyAccess.WrappedKey, "Wrapped key should not be empty") // Verify encryption information - assert.Equal(t, kGCMCipherAlgorithm, finalizeResult.Manifest.Method.Algorithm, "Algorithm should be AES-256-GCM") + assert.Equal(t, "AES-256-GCM", finalizeResult.Manifest.Method.Algorithm, "Algorithm should be AES-256-GCM") assert.True(t, finalizeResult.Manifest.Method.IsStreamable, "Should be marked as streamable") assert.NotEmpty(t, finalizeResult.Manifest.Policy, "Policy should not be empty") } @@ -385,19 +385,18 @@ func testMultiSegmentFlow(t *testing.T) { } // Write segments in order + results := make([]*SegmentResult, len(segments)) for i, data := range segments { - _, err := writer.WriteSegment(ctx, i, data) + res, err := writer.WriteSegment(ctx, i, data) require.NoError(t, err, "Failed to write segment %d", i) + results[i] = res } - // Verify all segments were recorded - assert.Len(t, writer.segments, 3, "Should have three segments") - assert.Equal(t, 2, writer.maxSegmentIndex, "Max segment index should be 2") - // Verify each segment for i, data := range segments { - assert.Equal(t, int64(len(data)), writer.segments[i].Size, "Segment %d size should match", i) - assert.NotEmpty(t, writer.segments[i].Hash, "Segment %d hash should be set", i) + assert.Equal(t, i, results[i].Index, "Segment %d index should match", i) + assert.Equal(t, int64(len(data)), results[i].PlaintextSize, "Segment %d size should match", i) + assert.NotEmpty(t, results[i].Hash, "Segment %d hash should be set", i) } // Finalize with attributes for proper key access setup @@ -410,6 +409,13 @@ func testMultiSegmentFlow(t *testing.T) { // Validate manifest against schema validateManifestSchema(t, finalizeResult.Manifest) + // Verify all segments were recorded + assert.Equal(t, 3, finalizeResult.TotalSegments, "Should have three segments") + require.Len(t, finalizeResult.Manifest.Segments, 3, "Manifest should describe three segments") + for i, data := range segments { + assert.Equal(t, int64(len(data)), finalizeResult.Manifest.Segments[i].Size, "Segment %d size should match", i) + } + // Verify root signature was calculated from all segments assert.NotEmpty(t, finalizeResult.Manifest.Signature, "Root signature should be set") assert.Equal(t, "HS256", finalizeResult.Manifest.Algorithm, "Root signature algorithm should be HS256") @@ -435,9 +441,6 @@ func testKeySplittingWithMultipleAttributes(t *testing.T) { } // Finalize with multiple attributes - originalDEK := make([]byte, len(writer.dek)) - copy(originalDEK, writer.dek) - finalizeResult, err := writer.Finalize(ctx, WithAttributeValues(attributes)) require.NoError(t, err, "Failed to finalize TDF with multiple attributes") @@ -463,9 +466,13 @@ func testKeySplittingWithMultipleAttributes(t *testing.T) { } } - // Test that we can theoretically reconstruct the key from splits - // (This verifies the XOR splitting logic worked correctly) - assert.Len(t, originalDEK, 32, "Original DEK should be 32 bytes") + // Three allOf attributes on distinct KAS must XOR-split into three shares, + // each carrying its own split ID; a share is useless on its own. + splitIDs := make(map[string]bool, len(keyAccessObjs)) + for _, keyAccess := range keyAccessObjs { + splitIDs[keyAccess.SplitID] = true + } + assert.Len(t, splitIDs, 3, "Each of the three allOf KAS should get its own split") } // testManifestGeneration tests detailed manifest structure and content @@ -511,7 +518,7 @@ func testManifestGeneration(t *testing.T) { // Verify encryption information encInfo := finalizeResult.Manifest.EncryptionInformation - assert.Equal(t, kGCMCipherAlgorithm, encInfo.Method.Algorithm, "Algorithm should be AES-256-GCM") + assert.Equal(t, "AES-256-GCM", encInfo.Method.Algorithm, "Algorithm should be AES-256-GCM") assert.True(t, encInfo.Method.IsStreamable, "Should be streamable") assert.NotEmpty(t, encInfo.Policy, "Policy should not be empty") @@ -671,20 +678,36 @@ func testErrorConditions(t *testing.T) { assert.Contains(t, err.Error(), "no default KAS", "Error should mention missing default KAS") }) - t.Run("EmptySegmentHash", func(t *testing.T) { + t.Run("SegmentsNamesUnwrittenIndex", func(t *testing.T) { writer, err := NewWriter(ctx) require.NoError(t, err) - // Manually corrupt segment hash to test error handling - writer.segments[0] = &Segment{Hash: "", Size: 10, EncryptedSize: 26} - writer.maxSegmentIndex = 0 + _, err = writer.WriteSegment(ctx, 0, []byte("first")) + require.NoError(t, err) + _, err = writer.WriteSegment(ctx, 5, []byte("sixth")) + require.NoError(t, err) + + attributes := []*policy.Value{ + createTestAttribute("https://example.com/attr/Test/value/Error", testKAS1, "kid1"), + } + _, err = writer.Finalize(ctx, WithAttributeValues(attributes), WithSegments([]int{0, 1})) + require.Error(t, err, "Should reject a segment that was never written") + assert.Contains(t, err.Error(), "not written", "Error should name the unwritten segment") + }) + + t.Run("SegmentZeroMissing", func(t *testing.T) { + writer, err := NewWriter(ctx) + require.NoError(t, err) + + _, err = writer.WriteSegment(ctx, 1, []byte("second")) + require.NoError(t, err) attributes := []*policy.Value{ createTestAttribute("https://example.com/attr/Test/value/Error", testKAS1, "kid1"), } _, err = writer.Finalize(ctx, WithAttributeValues(attributes)) - require.Error(t, err, "Should detect empty segment hash") - assert.Contains(t, err.Error(), "empty segment hash", "Error message should mention empty segment hash") + require.ErrorIs(t, err, ErrMissingSegmentZero, + "segment 0 carries the payload's ZIP local file header") }) } @@ -695,10 +718,6 @@ func testXORReconstruction(t *testing.T) { writer, err := NewWriter(ctx) require.NoError(t, err) - // Store original DEK for comparison - originalDEK := make([]byte, len(writer.dek)) - copy(originalDEK, writer.dek) - // Write test data _, err = writer.WriteSegment(ctx, 0, []byte("XOR test data")) require.NoError(t, err) @@ -731,9 +750,6 @@ func testXORReconstruction(t *testing.T) { require.NoError(t, err, "Should be able to decode wrapped key") assert.NotEmpty(t, wrappedKeyBytes, "Decoded wrapped key should not be empty") } - - // Verify the original DEK is the expected size - assert.Len(t, originalDEK, 32, "Original DEK should be 32 bytes") } // testDifferentAttributeRules tests TDF creation with different attribute rule types @@ -796,22 +812,23 @@ func testOutOfOrderSegments(t *testing.T) { require.NoError(t, err, "Failed to write segment %d", idx) } - // Verify all segments are present and in correct positions - assert.Len(t, writer.segments, 3, "Should have three segments") - assert.Equal(t, 2, writer.maxSegmentIndex, "Max segment index should be 2") - - for i := 0; i < 3; i++ { - assert.Equal(t, int64(len(segments[i])), writer.segments[i].Size, "Segment %d size should match", i) - assert.NotEmpty(t, writer.segments[i].Hash, "Segment %d should have hash", i) - } - // Finalize with attributes attributes := []*policy.Value{ createTestAttribute("https://example.com/attr/Order/value/Test", testKAS1, "kid1"), } finalizeResult, err := writer.Finalize(ctx, WithAttributeValues(attributes)) require.NoError(t, err, "Should finalize successfully with out-of-order segments") - assert.NotNil(t, finalizeResult.Manifest, "Manifest should be created") + require.NotNil(t, finalizeResult.Manifest, "Manifest should be created") + + // Whatever order they were written in, the manifest must describe the + // segments in ascending index order -- that is the order a reader + // concatenates the payload in. + assert.Equal(t, 3, finalizeResult.TotalSegments, "Should have three segments") + require.Len(t, finalizeResult.Manifest.Segments, 3, "Manifest should describe three segments") + for i := range 3 { + assert.Equal(t, int64(len(segments[i])), finalizeResult.Manifest.Segments[i].Size, "Segment %d size should match", i) + assert.NotEmpty(t, finalizeResult.Manifest.Segments[i].Hash, "Segment %d should have hash", i) + } // Validate manifest against schema validateManifestSchema(t, finalizeResult.Manifest) @@ -894,12 +911,18 @@ func createTestAttributeWithAlgorithm(t *testing.T, fqn, kasURL, kid string, alg return value } -// hybridUnwrapForTest base64-decodes the wrappedKey from a manifest KAO and -// unwraps it with the matching hybrid private key, asserting the recovered DEK -// exactly matches the writer DEK. -func hybridUnwrapForTest(t *testing.T, ktype ocrypto.KeyType, privatePEM, wrappedKeyB64 string, expectedDEK []byte) { +// hybridUnwrapForTest unwraps a manifest KAO's wrappedKey with the matching +// hybrid private key and asserts the recovered key really is the DEK the +// writer used. +// +// The writer does not expose its DEK, so the check goes through the policy +// binding instead: it is an HMAC-SHA256 over the base64 policy keyed by the +// split's key, and a single-KAS TDF has exactly one split holding the whole +// DEK. Reproducing the manifest's binding from the recovered bytes therefore +// proves they match. +func hybridUnwrapForTest(t *testing.T, ktype ocrypto.KeyType, privatePEM string, manifest *Manifest, keyAccess KeyAccess) { t.Helper() - wrappedDER, err := ocrypto.Base64Decode([]byte(wrappedKeyB64)) + wrappedDER, err := ocrypto.Base64Decode([]byte(keyAccess.WrappedKey)) require.NoError(t, err, "Base64Decode wrapped key") dec, err := ocrypto.FromPrivatePEM(privatePEM) @@ -914,7 +937,14 @@ func hybridUnwrapForTest(t *testing.T, ktype ocrypto.KeyType, privatePEM, wrappe dek, err := dec.Decrypt(wrappedDER) require.NoError(t, err, "hybrid Decrypt") - assert.Equal(t, expectedDEK, dek, "%s recovered DEK", ktype) + require.Len(t, dek, 32, "%s recovered DEK should be 32 bytes", ktype) + + hash := hex.EncodeToString(ocrypto.CalculateSHA256Hmac(dek, []byte(manifest.Policy))) + expected := string(ocrypto.Base64Encode([]byte(hash))) + + binding, ok := keyAccess.PolicyBinding.(PolicyBinding) + require.True(t, ok, "policy binding should be a PolicyBinding, got %T", keyAccess.PolicyBinding) + assert.Equal(t, expected, binding.Hash, "%s recovered DEK should reproduce the policy binding", ktype) } func testHybridXWingFlow(t *testing.T) { @@ -929,7 +959,6 @@ func testHybridXWingFlow(t *testing.T) { writer, err := NewWriter(ctx) require.NoError(t, err) - expectedDEK := append([]byte(nil), writer.dek...) _, err = writer.WriteSegment(ctx, 0, []byte("hybrid xwing test data")) require.NoError(t, err) @@ -951,7 +980,7 @@ func testHybridXWingFlow(t *testing.T) { assert.NotEmpty(t, keyAccess.WrappedKey) validateManifestSchema(t, result.Manifest) - hybridUnwrapForTest(t, ocrypto.HybridXWingKey, privPEM, keyAccess.WrappedKey, expectedDEK) + hybridUnwrapForTest(t, ocrypto.HybridXWingKey, privPEM, result.Manifest, keyAccess) } func testHybridP256MLKEM768Flow(t *testing.T) { @@ -966,7 +995,6 @@ func testHybridP256MLKEM768Flow(t *testing.T) { writer, err := NewWriter(ctx) require.NoError(t, err) - expectedDEK := append([]byte(nil), writer.dek...) _, err = writer.WriteSegment(ctx, 0, []byte("hybrid p256 mlkem768 test data")) require.NoError(t, err) @@ -988,7 +1016,7 @@ func testHybridP256MLKEM768Flow(t *testing.T) { assert.NotEmpty(t, keyAccess.WrappedKey) validateManifestSchema(t, result.Manifest) - hybridUnwrapForTest(t, ocrypto.HybridSecp256r1MLKEM768Key, privPEM, keyAccess.WrappedKey, expectedDEK) + hybridUnwrapForTest(t, ocrypto.HybridSecp256r1MLKEM768Key, privPEM, result.Manifest, keyAccess) } func testHybridP384MLKEM1024Flow(t *testing.T) { @@ -1003,7 +1031,6 @@ func testHybridP384MLKEM1024Flow(t *testing.T) { writer, err := NewWriter(ctx) require.NoError(t, err) - expectedDEK := append([]byte(nil), writer.dek...) _, err = writer.WriteSegment(ctx, 0, []byte("hybrid p384 mlkem1024 test data")) require.NoError(t, err) @@ -1025,7 +1052,7 @@ func testHybridP384MLKEM1024Flow(t *testing.T) { assert.NotEmpty(t, keyAccess.WrappedKey) validateManifestSchema(t, result.Manifest) - hybridUnwrapForTest(t, ocrypto.HybridSecp384r1MLKEM1024Key, privPEM, keyAccess.WrappedKey, expectedDEK) + hybridUnwrapForTest(t, ocrypto.HybridSecp384r1MLKEM1024Key, privPEM, result.Manifest, keyAccess) } // validateManifestSchema validates a TDF manifest against the JSON schema @@ -1161,8 +1188,8 @@ func testGetManifestBeforeAndAfterFinalize(t *testing.T) { require.NoError(t, err) require.NotNil(t, m0) assert.Equal(t, TDFSpecVersion, m0.TDFVersion) - assert.Equal(t, tdfAsZip, m0.Protocol) - assert.Equal(t, tdfZipReference, m0.Type) + assert.Equal(t, "zip", m0.Protocol) + assert.Equal(t, "reference", m0.Type) assert.True(t, m0.IsEncrypted) // No segments yet assert.Empty(t, m0.Segments)