From bac467c92cc1d345a46c5268a5c69ad5a5dfc2af Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Mon, 31 Aug 2026 22:23:01 -0400 Subject: [PATCH] chore(sdk): rewrite CreateTDF on top of the chunked writer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CreateTDFContext and the chunked writer had grown two full implementations of the same thing: build a policy, split the DEK, wrap each share to its KAS, encrypt segments, accumulate an aggregate hash, sign assertions, emit a manifest. Two copies of TDF construction is one too many — every spec change has to land twice, and the second copy is the one that gets forgotten. CreateTDF now delegates. It keeps the parts that are genuinely its own — the KAO template, autoconfigure, the input-size resolution and read loop from the previous commit — and hands each segment to the chunked writer, which owns manifest assembly for both paths from here on. The two paths differ in when key access is resolved, so that is what the writer is now parameterized on. The chunked writer defers to a KeySplitter at Finalize, because a caller may still be adding attributes while segments are in flight. CreateTDF cannot: it knows its attributes up front and wants an unreachable KAS to fail the call before a single payload byte reaches the output writer. Both are expressed as a keyAccessResolver, with the DEK now injectable so CreateTDF can wrap it ahead of time and hand the writer a staticKeyAccess. Everything downstream of that split is shared: resolvePolicyAndKeyAccess and buildKeyAccessObjects replace prepareManifest's inline loop and the chunked writer's buildChunkedPolicy/buildChunkedKeyAccessObjects, so both paths now emit byte-identical policy and key access for the same attributes. Two things fall out of the unification, both moving the chunked writer onto the shipped classic behavior: - With zero attributes the policy body's "dataAttributes" and "dissem" are now null rather than []. createPolicyObjectFromFQNs initializes them inside the attribute loop; the deleted buildChunkedPolicy did so unconditionally. The classic path has always emitted null here. - A KAS named by a split but missing a public key is still rejected rather than skipped (the check moved into buildKeyAccessObjects), but the error now names the missing PEM rather than the absent map entry — the two cases were indistinguishable in practice and only the outcome matters. The writer also gained an explicit segment size. It previously reported the first segment's actual length as defaultSegmentSize, which is only correct when every segment is full; a single-segment TDF would advertise a short default. CreateTDF knows the configured size and now says so. TDFObject loses aesGcm and payloadKey, which only ever existed to carry state between prepareManifest and the encrypt loop. Deliberately not in this commit: removing enableEncryption, tdfFormat, and readActionName, which are dead but unrelated; they are a separate cleanup. Testing: the existing TDFSuite round trips pin byte-level output across integrity algorithms, segment sizes, target modes, and multi-KAS splits, and pass unchanged. Also verified against the streaming-input and input-size coverage added in the previous commit, the experimental chunked writer suite, and cross-module builds of examples, otdfctl, service, and tests-bdd. Signed-off-by: Dave Mihalcik --- sdk/chunked_test.go | 22 ++- sdk/chunked_writer.go | 136 +++++++--------- sdk/key_splitter.go | 179 +++++++++++++++++++-- sdk/tdf.go | 344 ++++++++++++++++------------------------ sdk/tdf_helpers_test.go | 10 +- 5 files changed, 371 insertions(+), 320 deletions(-) diff --git a/sdk/chunked_test.go b/sdk/chunked_test.go index 4079337a8d..285431e7a8 100644 --- a/sdk/chunked_test.go +++ b/sdk/chunked_test.go @@ -753,19 +753,17 @@ func TestChunkedECKeyAccess(t *testing.T) { for i := range dek { dek[i] = byte(i) } - splits := &SplitResult{ - KASPublicKeys: map[string]KASPublicKey{ - kasURL: { - Algorithm: string(ocrypto.EC256Key), - KID: "ec-kid", - PEM: pubPEM, - URL: kasURL, - }, - }, - Splits: []Split{{Data: dek, KASURLs: []string{kasURL}}}, - } + shares := []splitShare{{ + data: dek, + kases: []KASInfo{{ + URL: kasURL, + PublicKey: pubPEM, + KID: "ec-kid", + Algorithm: string(ocrypto.EC256Key), + }}, + }} - kaos, err := buildChunkedKeyAccessObjects(splits, []byte(`{"uuid":"test"}`), "") + kaos, err := buildKeyAccessObjects(shares, `{"uuid":"test"}`, "") require.NoError(t, err) require.Len(t, kaos, 1) diff --git a/sdk/chunked_writer.go b/sdk/chunked_writer.go index 79678a790d..95b81067e1 100644 --- a/sdk/chunked_writer.go +++ b/sdk/chunked_writer.go @@ -14,7 +14,6 @@ import ( "sync" "time" - "github.com/google/uuid" "github.com/opentdf/platform/lib/ocrypto" "github.com/opentdf/platform/protocol/go/policy" "github.com/opentdf/platform/sdk/internal/zipstream" @@ -232,6 +231,11 @@ type ChunkedWriterConfig struct { // fixedClock for deterministic ZIP output. clock clock + // dek is a pre-generated Data Encryption Key. When nil the writer + // draws one from rand. SDK.CreateTDF presets it so that it can + // resolve key access before emitting any payload bytes. + dek []byte + // excludeVersion omits the schemaVersion field from the manifest. // Set together with useHex by WithChunkedTargetMode; readers use // the field's absence as the pre-4.3.0 marker, so the two must @@ -250,6 +254,10 @@ type ChunkedWriterConfig struct { // signature. Defaults to HS256. integrityAlgorithm IntegrityAlgorithm + // keyAccess resolves the manifest policy and key access objects. + // Defaults to a splitterKeyAccess over splitter. + keyAccess keyAccessResolver + // rand is the entropy source used to generate the DEK. Defaults // to crypto/rand.Reader. rand io.Reader @@ -258,8 +266,14 @@ type ChunkedWriterConfig struct { // integrity hashes. Defaults to HS256. segmentIntegrityAlgorithm IntegrityAlgorithm + // segmentSize is the plaintext segment size advertised in the + // manifest. Zero means "report the first segment's actual size", + // which is right when every segment is the same length. + segmentSize int64 + // splitter maps attribute values to DEK splits at Finalize time. - // Defaults to DefaultKeySplitter (single-KAS only). + // Defaults to DefaultKeySplitter (single-KAS only). Ignored when + // keyAccess is set. splitter KeySplitter // useHex hex-encodes segment, root, and assertion signatures @@ -352,6 +366,10 @@ type chunkedWriter struct { // integrityAlgorithm is used for the root signature. integrityAlgorithm IntegrityAlgorithm + // keyAccess resolves the manifest policy and key access objects + // for the DEK. + keyAccess keyAccessResolver + // manifest holds the finalized manifest for post-Finalize // GetManifest calls. manifest *Manifest @@ -365,9 +383,9 @@ type chunkedWriter struct { // segments records per-index Segment metadata (hash + sizes). segments map[int]*Segment - // splitter converts attributes + DEK into key splits at - // Finalize time. - splitter KeySplitter + // segmentSize is the plaintext segment size to advertise in the + // manifest, or zero to infer it from the first segment. + segmentSize int64 // useHex selects the pre-4.3.0 doubly-encoded signature form. // Read by WriteSegment, so it is fixed at construction rather @@ -401,15 +419,30 @@ func NewChunkedWriter(_ context.Context, opts ...ChunkedWriterOption) (ChunkedWr return nil, err } } + return newChunkedWriter(cfg) +} - dek := make([]byte, kKeySize) - if _, err := io.ReadFull(cfg.rand, dek); err != nil { - return nil, fmt.Errorf("generate DEK: %w", err) +// newChunkedWriter builds the writer from a fully-populated config. +// SDK.CreateTDF calls this directly with the unexported knobs its +// classic behavior needs — a preset DEK, key access resolved before +// the first payload byte, a fixed segment size — rather than going +// through the public option set. +func newChunkedWriter(cfg ChunkedWriterConfig) (*chunkedWriter, error) { + dek := cfg.dek + if dek == nil { + dek = make([]byte, kKeySize) + if _, err := io.ReadFull(cfg.rand, dek); err != nil { + return nil, fmt.Errorf("generate DEK: %w", err) + } } block, err := cfg.cipherFactory(dek) if err != nil { return nil, fmt.Errorf("build segment cipher: %w", err) } + keyAccess := cfg.keyAccess + if keyAccess == nil { + keyAccess = splitterKeyAccess{splitter: cfg.splitter} + } return &chunkedWriter{ archiveWriter: cfg.archiveFactory(cfg.clock), block: block, @@ -418,9 +451,10 @@ func NewChunkedWriter(_ context.Context, opts ...ChunkedWriterOption) (ChunkedWr initialAttributes: cfg.initialAttributes, initialDefaultKAS: cfg.initialDefaultKAS, integrityAlgorithm: cfg.integrityAlgorithm, + keyAccess: keyAccess, segmentIntegrityAlgorithm: cfg.segmentIntegrityAlgorithm, segments: make(map[int]*Segment), - splitter: cfg.splitter, + segmentSize: cfg.segmentSize, useHex: cfg.useHex, }, nil } @@ -716,18 +750,10 @@ func (w *chunkedWriter) snapshotLocked(keep []int) (*chunkedSnapshot, error) { // buildManifest assembles the manifest from a snapshot. It reads no // mutable writer state and holds no lock: every other field it touches -// (dek, splitter, the integrity algorithms, useHex) is fixed at +// (dek, keyAccess, the integrity algorithms, useHex) is fixed at // construction. func (w *chunkedWriter) buildManifest(ctx context.Context, cfg *ChunkedFinalizeConfig, snap *chunkedSnapshot) (*Manifest, int64, int64, error) { - splits, err := w.splitter.Split(ctx, cfg.attributes, w.dek, cfg.defaultKAS) - if err != nil { - return nil, 0, 0, err - } - policyBytes, err := buildChunkedPolicy(cfg.attributes) - if err != nil { - return nil, 0, 0, err - } - kaos, err := buildChunkedKeyAccessObjects(splits, policyBytes, cfg.encryptedMetadata) + base64Policy, kaos, err := w.keyAccess.resolve(ctx, w.dek, cfg) if err != nil { return nil, 0, 0, err } @@ -735,7 +761,7 @@ func (w *chunkedWriter) buildManifest(ctx context.Context, cfg *ChunkedFinalizeC encInfo := EncryptionInformation{ KeyAccessObjs: kaos, KeyAccessType: kSplitKeyType, - Policy: string(ocrypto.Base64Encode(policyBytes)), + Policy: base64Policy, Method: Method{ Algorithm: kGCMCipherAlgorithm, IsStreamable: true, @@ -758,7 +784,15 @@ func (w *chunkedWriter) buildManifest(ctx context.Context, cfg *ChunkedFinalizeC } aggregate.Write(decoded) } - if len(snap.segments) > 0 { + // A caller that knows the segment size says so, because the first + // segment's actual length is only the right answer when every + // segment is full — and the last one usually is not, so a + // single-segment TDF would otherwise advertise a short default. + switch { + case w.segmentSize > 0: + encInfo.DefaultSegmentSize = w.segmentSize + encInfo.DefaultEncryptedSegSize = w.segmentSize + gcmIvSize + aesBlockSize + case len(snap.segments) > 0: encInfo.DefaultEncryptedSegSize = snap.segments[0].EncryptedSize encInfo.DefaultSegmentSize = snap.segments[0].Size } @@ -852,66 +886,6 @@ func (w *chunkedWriter) segmentOrderLocked(keep []int) ([]int, error) { return out, nil } -// buildChunkedKeyAccessObjects wraps each split share to each KAS -// listed by the splitter. -func buildChunkedKeyAccessObjects(splits *SplitResult, policyBytes []byte, metadata string) ([]KeyAccess, error) { - if splits == nil || len(splits.Splits) == 0 { - return nil, errors.New("no splits produced") - } - base64Policy := ocrypto.Base64Encode(policyBytes) - - var out []KeyAccess - for _, split := range splits.Splits { - // Policy binding and metadata are keyed on the split share, not - // on the KAS, so compute them once per split rather than once - // per KAS URL in an OR-group. - policyBinding := createPolicyBinding(split.Data, base64Policy) - var encMeta string - if metadata != "" { - m, err := encryptMetadata(split.Data, metadata) - if err != nil { - return nil, fmt.Errorf("encrypt metadata for split %s: %w", split.ID, err) - } - encMeta = m - } - for _, url := range split.KASURLs { - // A KAS named by a split but absent from KASPublicKeys is an - // error, not something to skip. Dropping it silently removes - // the only KAO that would have let that KAS unwrap this - // share; if every URL on the split is missing, the share - // becomes unrecoverable and the TDF undecryptable, with - // nothing in the output to say why. - pk, ok := splits.KASPublicKeys[url] - if !ok || pk.PEM == "" { - return nil, fmt.Errorf("splitID:[%s], kas:[%s]: %w", split.ID, url, errKasPubKeyMissing) - } - kao, err := createKeyAccess(pk.toKASInfo(), split.Data, policyBinding, encMeta, split.ID) - if err != nil { - return nil, fmt.Errorf("wrap key for %s: %w", url, err) - } - out = append(out, kao) - } - } - if len(out) == 0 { - return nil, errors.New("no valid key access objects generated") - } - return out, nil -} - -// buildChunkedPolicy composes the TDF Policy document from attribute -// values. -func buildChunkedPolicy(values []*policy.Value) ([]byte, error) { - p := PolicyObject{UUID: uuid.NewString()} - p.Body.DataAttributes = make([]attributeObject, 0, len(values)) - p.Body.Dissem = make([]string, 0) - for _, v := range values { - p.Body.DataAttributes = append(p.Body.DataAttributes, attributeObject{ - Attribute: v.GetFqn(), - }) - } - return json.Marshal(p) -} - // cloneChunkedManifest returns a shallow-deep copy safe to hand out. func cloneChunkedManifest(in *Manifest) *Manifest { if in == nil { diff --git a/sdk/key_splitter.go b/sdk/key_splitter.go index a40fded1f3..470fea684a 100644 --- a/sdk/key_splitter.go +++ b/sdk/key_splitter.go @@ -2,9 +2,12 @@ package sdk import ( "context" + "encoding/json" "errors" "fmt" + "io" + "github.com/opentdf/platform/lib/ocrypto" "github.com/opentdf/platform/protocol/go/policy" ) @@ -73,18 +76,6 @@ type KASPublicKey struct { URL string } -// toKASInfo adapts the splitter's wrapping-key descriptor to the -// KASInfo shape consumed by createKeyAccess. Default is not carried -// over; it plays no part in building a key access object. -func (k KASPublicKey) toKASInfo() KASInfo { - return KASInfo{ - URL: k.URL, - PublicKey: k.PEM, - KID: k.KID, - Algorithm: k.Algorithm, - } -} - // ErrSplitterRequiresDefaultKAS is returned by the default key // splitter when no default KAS was supplied. The default splitter is // single-KAS only; multi-attribute splits require injecting a full @@ -155,6 +146,170 @@ func (s *singleKASSplitter) Split(_ context.Context, _ []*policy.Value, dek []by }, nil } +// keyAccessResolver turns a DEK into the two manifest fields that +// bind it to policy: the base64-encoded policy object and the key +// access objects wrapping the DEK to each KAS. The writer holds one; +// SDK.CreateTDF resolves its key access up front and supplies a +// staticKeyAccess, while the chunked path defers to a KeySplitter at +// Finalize time. +type keyAccessResolver interface { + resolve(ctx context.Context, dek []byte, cfg *ChunkedFinalizeConfig) (string, []KeyAccess, error) +} + +// splitShare is one XOR share of the DEK together with every KAS able +// to unwrap it. Several KAS entries on one share mean any of them +// suffices (OR semantics); several shares mean all are required (AND). +type splitShare struct { + // id names the share in the manifest ("sid"). Empty when the TDF + // has a single share. + id string + + // data is the share itself. + data []byte + + // kases are the wrapping targets for this share. + kases []KASInfo +} + +// staticKeyAccess returns key access objects resolved ahead of time. +type staticKeyAccess struct { + // kaos are the pre-built key access objects. + kaos []KeyAccess + + // policy is the base64-encoded policy object the kaos are bound to. + policy string +} + +func (r staticKeyAccess) resolve(_ context.Context, _ []byte, _ *ChunkedFinalizeConfig) (string, []KeyAccess, error) { + return r.policy, r.kaos, nil +} + +// splitterKeyAccess adapts a public KeySplitter to keyAccessResolver. +type splitterKeyAccess struct { + // splitter maps attributes plus the DEK onto KAS-addressed shares. + splitter KeySplitter +} + +func (r splitterKeyAccess) resolve(ctx context.Context, dek []byte, cfg *ChunkedFinalizeConfig) (string, []KeyAccess, error) { + splits, err := r.splitter.Split(ctx, cfg.attributes, dek, cfg.defaultKAS) + if err != nil { + return "", nil, err + } + if splits == nil || len(splits.Splits) == 0 { + return "", nil, errors.New("no splits produced") + } + + shares := make([]splitShare, 0, len(splits.Splits)) + for _, split := range splits.Splits { + share := splitShare{id: split.ID, data: split.Data} + for _, url := range split.KASURLs { + // A KAS named by a split but absent from KASPublicKeys + // yields an empty PEM here, which buildKeyAccessObjects + // rejects. Carrying it through rather than skipping it is + // deliberate: dropping it silently removes the only KAO + // that would have let that KAS unwrap this share, and if + // every URL on the split is missing, the share becomes + // unrecoverable and the TDF undecryptable with nothing in + // the output to say why. + pk := splits.KASPublicKeys[url] + share.kases = append(share.kases, KASInfo{ + URL: url, + PublicKey: pk.PEM, + KID: pk.KID, + Algorithm: pk.Algorithm, + }) + } + shares = append(shares, share) + } + + fqns := make([]string, 0, len(cfg.attributes)) + for _, v := range cfg.attributes { + fqns = append(fqns, v.GetFqn()) + } + return resolvePolicyAndKeyAccess(fqns, shares, cfg.encryptedMetadata) +} + +// resolvePolicyAndKeyAccess builds the policy document the DEK is bound to and wraps +// every share to its KAS targets, returning the two manifest fields that bind a DEK to +// policy. Shared by SDK.CreateTDF's KAO template path and the chunked writer's +// KeySplitter path, so both emit byte-identical policy for the same attributes. +func resolvePolicyAndKeyAccess(fqns []string, shares []splitShare, metadata string) (string, []KeyAccess, error) { + policyObj, err := createPolicyObjectFromFQNs(fqns) + if err != nil { + return "", nil, fmt.Errorf("fail to create policy object:%w", err) + } + policyObjectAsStr, err := json.Marshal(policyObj) + if err != nil { + return "", nil, fmt.Errorf("json.Marshal failed:%w", err) + } + base64Policy := string(ocrypto.Base64Encode(policyObjectAsStr)) + + kaos, err := buildKeyAccessObjects(shares, base64Policy, metadata) + if err != nil { + return "", nil, err + } + return base64Policy, kaos, nil +} + +// buildKeyAccessObjects wraps every share to each of its KAS targets, +// emitting the manifest's keyAccess array in share order. +func buildKeyAccessObjects(shares []splitShare, base64Policy, metadata string) ([]KeyAccess, error) { + var kaos []KeyAccess + for _, share := range shares { + // Policy binding and metadata are keyed on the split share, not + // on the KAS, so compute them once per share rather than once + // per KAS URL in an OR-group. + policyBinding := createPolicyBinding(share.data, base64Policy) + + var encryptedMetadata string + if metadata != "" { + var err error + encryptedMetadata, err = encryptMetadata(share.data, metadata) + if err != nil { + return nil, err + } + } + + for _, kasInfo := range share.kases { + if kasInfo.PublicKey == "" { + return nil, fmt.Errorf("splitID:[%s], kas:[%s]: %w", share.id, kasInfo.URL, errKasPubKeyMissing) + } + keyAccess, err := createKeyAccess(kasInfo, share.data, policyBinding, encryptedMetadata, share.id) + if err != nil { + return nil, err + } + kaos = append(kaos, keyAccess) + } + } + if len(kaos) == 0 { + return nil, errors.New("no key access objects generated") + } + return kaos, nil +} + +// splitDEK returns count XOR shares of dek. Every share but the last +// is random; the last absorbs the parity so the shares XOR back to dek. +func splitDEK(dek []byte, count int, rand io.Reader) ([][]byte, error) { + if count <= 0 { + return nil, errors.New("no key splits requested") + } + shares := make([][]byte, count) + parity := make([]byte, len(dek)) + copy(parity, dek) + for i := range count - 1 { + share := make([]byte, len(dek)) + if _, err := io.ReadFull(rand, share); err != nil { + return nil, fmt.Errorf("generate key split failed: %w", err) + } + for j, b := range share { + parity[j] ^= b + } + shares[i] = share + } + shares[count-1] = parity + return shares, nil +} + // algorithmPolicyToString maps a policy.Algorithm enum to the // ocrypto.KeyType string form used when picking a wrap scheme. // Unknown enums, including ALGORITHM_UNSPECIFIED, return the empty diff --git a/sdk/tdf.go b/sdk/tdf.go index 30db9f5ae9..f2efc3f0ae 100644 --- a/sdk/tdf.go +++ b/sdk/tdf.go @@ -8,10 +8,10 @@ import ( "encoding/json" "errors" "fmt" - "hash/crc32" "io" "log/slog" "net/http" + "slices" "strconv" "strings" @@ -76,10 +76,8 @@ type RequiredObligations struct { } type TDFObject struct { - manifest Manifest - size int64 - aesGcm ocrypto.AesGcm - payloadKey [kKeySize]byte + manifest Manifest + size int64 } type countingWriter struct { @@ -179,12 +177,6 @@ func (s SDK) CreateTDFContext(ctx context.Context, writer io.Writer, reader io.R return nil, err } - tdfObject := &TDFObject{} - err = s.prepareManifest(ctx, tdfObject, *tdfConfig) - if err != nil { - return nil, fmt.Errorf("fail to create a new split key: %w", err) - } - segmentSize := tdfConfig.defaultSegmentSize if segmentSize > maxSegmentSize { return nil, fmt.Errorf("segment size too large: %d", segmentSize) @@ -198,41 +190,27 @@ func (s SDK) CreateTDFContext(ctx context.Context, writer io.Writer, reader io.R } totalSegments := segmentCount(inputSize, segmentSize) - encryptedSegmentSize := segmentSize + gcmIvSize + aesBlockSize - - // The ZIP64 choice is baked into the payload's local file header, which goes out - // ahead of the first segment, so it cannot be revisited once the archive has - // started. Reserve ZIP64 for payloads that a 32-bit offset cannot address — and - // for payloads of unknown length, which might turn out to be one. - payloadSize := inputSize + int64(totalSegments)*(gcmIvSize+aesBlockSize) - zipMode := zipstream.Zip64Auto - if inputSize == inputSizeUnknown || payloadSize >= zip64MagicVal { - zipMode = zipstream.Zip64Always - } - - archiveOpts := []zipstream.Option{zipstream.WithZip64Mode(zipMode)} - if totalSegments > 0 { - archiveOpts = append(archiveOpts, zipstream.WithMaxSegments(totalSegments)) + chunked, err := s.newTDFChunkedWriter(ctx, tdfConfig, inputSize, totalSegments) + if err != nil { + return nil, err } - archiveWriter := zipstream.NewSegmentTDFWriter(totalSegments, archiveOpts...) outputWriter := &countingWriter{writer: writer} // A known length doubles as a read limit: overrunning it would invalidate the - // ZIP64 choice made from it above. Only as large as the payload actually needs, - // too — the segment size defaults to 2 MiB, so sizing on it alone would allocate - // that much to encrypt a handful of bytes. The buffer never shrinks to zero, so a - // read that comes back empty always means EOF. + // ZIP64 choice made from it in newTDFChunkedWriter. Only as large as the payload + // actually needs, too — the segment size defaults to 2 MiB, so sizing on it alone + // would allocate that much to encrypt a handful of bytes. The buffer never shrinks + // to zero, so a read that comes back empty always means EOF. readBufSize := segmentSize if inputSize != inputSizeUnknown { reader = io.LimitReader(reader, inputSize) readBufSize = max(1, min(segmentSize, inputSize)) } - var aggregateHashBuilder strings.Builder var bytesRead int64 readBuf := make([]byte, readBufSize) - for segmentIndex := 0; ; segmentIndex++ { + for index := 0; ; index++ { // io.Reader.Read is free to return fewer bytes than asked for without // erroring, so a bare Read would cut segments short at the whim of the // reader. ReadFull retries until the segment is filled or the input runs @@ -243,50 +221,19 @@ func (s SDK) CreateTDFContext(ctx context.Context, writer io.Writer, reader io.R } // A payload whose length is an exact multiple of the segment size reports // EOF with nothing read. An empty payload still gets one empty segment. - if n == 0 && segmentIndex > 0 { + if n == 0 && index > 0 { break } - readSize := int64(n) - bytesRead += readSize + bytesRead += int64(n) - cipherData, err := tdfObject.aesGcm.Encrypt(readBuf[:readSize]) + segment, err := chunked.WriteSegment(ctx, index, readBuf[:n]) if err != nil { - return nil, fmt.Errorf("ocrypto.AesGcm.Encrypt failed: %w", err) - } - - crc := crc32.ChecksumIEEE(cipherData) - headerBytes, err := archiveWriter.WriteSegment(ctx, segmentIndex, uint64(len(cipherData)), crc) - if err != nil { - return nil, fmt.Errorf("zipstream.WriteSegment failed: %w", err) - } - - if len(headerBytes) > 0 { - _, err = outputWriter.Write(headerBytes) - if err != nil { - return nil, fmt.Errorf("io.writer.Write failed: %w", err) - } + return nil, err } - - _, err = outputWriter.Write(cipherData) - if err != nil { + if _, err := io.Copy(outputWriter, segment.TDFData); err != nil { return nil, fmt.Errorf("io.writer.Write failed: %w", err) } - segmentSig, err := calculateSignature(cipherData, tdfObject.payloadKey[:], - tdfConfig.segmentIntegrityAlgorithm, tdfConfig.useHex) - if err != nil { - return nil, fmt.Errorf("splitKey.GetSignaturefailed: %w", err) - } - - aggregateHashBuilder.WriteString(segmentSig) - segmentInfo := Segment{ - Hash: string(ocrypto.Base64Encode([]byte(segmentSig))), - Size: readSize, - EncryptedSize: int64(len(cipherData)), - } - - tdfObject.manifest.Segments = append(tdfObject.manifest.Segments, segmentInfo) - if readErr != nil { break } @@ -300,76 +247,97 @@ func (s SDK) CreateTDFContext(ctx context.Context, writer io.Writer, reader io.R return nil, fmt.Errorf("%w: read %d of %d bytes", errInputShorterThanDeclared, bytesRead, inputSize) } - rootSignature, err := calculateSignature([]byte(aggregateHashBuilder.String()), tdfObject.payloadKey[:], - tdfConfig.integrityAlgorithm, tdfConfig.useHex) + finalizeOpts, err := tdfFinalizeOptions(tdfConfig) if err != nil { - return nil, fmt.Errorf("splitKey.GetSignaturefailed: %w", err) + return nil, err } - sig := string(ocrypto.Base64Encode([]byte(rootSignature))) - tdfObject.manifest.Signature = sig + result, err := chunked.Finalize(ctx, finalizeOpts...) + if err != nil { + return nil, err + } - tdfObject.manifest.Algorithm = integrityAlgorithmString(tdfConfig.integrityAlgorithm) + if _, err := outputWriter.Write(result.Data); err != nil { + return nil, fmt.Errorf("io.writer.Write failed: %w", err) + } - tdfObject.manifest.DefaultSegmentSize = segmentSize - tdfObject.manifest.DefaultEncryptedSegSize = encryptedSegmentSize + return &TDFObject{ + manifest: *result.Manifest, + size: outputWriter.written, + }, nil +} - tdfObject.manifest.SegmentHashAlgorithm = integrityAlgorithmString(tdfConfig.segmentIntegrityAlgorithm) - tdfObject.manifest.Method.IsStreamable = true +// newTDFChunkedWriter builds the chunked writer backing SDK.CreateTDF. Key access is +// resolved here, before any payload byte is written, so that an unreachable KAS fails +// the call without leaving a partial TDF on the output writer. +// +// inputSize may be inputSizeUnknown and totalSegments zero, for a payload that can only +// be measured by reading it. +func (s SDK) newTDFChunkedWriter(ctx context.Context, tdfConfig *TDFConfig, inputSize int64, totalSegments int) (*chunkedWriter, error) { + dek := make([]byte, kKeySize) + if _, err := io.ReadFull(defaultRand, dek); err != nil { + return nil, fmt.Errorf("fail to create a new split key: %w", err) + } - // add payload info - mimeType := tdfConfig.mimeType - if mimeType == "" { - mimeType = defaultMimeType + base64Policy, kaos, err := s.resolveKeyAccess(ctx, tdfConfig, dek) + if err != nil { + return nil, fmt.Errorf("fail to create a new split key: %w", err) } - tdfObject.manifest.MimeType = mimeType - tdfObject.manifest.Protocol = tdfAsZip - tdfObject.manifest.Type = tdfZipReference - tdfObject.manifest.URL = zipstream.TDFPayloadFileName - tdfObject.manifest.IsEncrypted = true + // The ZIP64 choice is baked into the payload's local file header, which goes out + // with segment 0, so it cannot be revisited once the archive has started. Reserve + // ZIP64 for payloads that a 32-bit offset cannot address — and for payloads of + // unknown length, which might turn out to be one. + payloadSize := inputSize + int64(totalSegments)*(gcmIvSize+aesBlockSize) + zipMode := zipstream.Zip64Auto + if inputSize == inputSizeUnknown || payloadSize >= zip64MagicVal { + zipMode = zipstream.Zip64Always + } + + return newChunkedWriter(ChunkedWriterConfig{ + archiveFactory: func(c clock) zipstream.SegmentWriter { + archiveOpts := []zipstream.Option{ + zipstream.WithZip64Mode(zipMode), + zipstream.WithClock(c.Now), + } + if totalSegments > 0 { + archiveOpts = append(archiveOpts, zipstream.WithMaxSegments(totalSegments)) + } + return zipstream.NewSegmentTDFWriter(totalSegments, archiveOpts...) + }, + cipherFactory: defaultSegmentCipherFactory, + clock: systemClock{}, + dek: dek, + integrityAlgorithm: tdfConfig.integrityAlgorithm, + keyAccess: staticKeyAccess{kaos: kaos, policy: base64Policy}, + segmentIntegrityAlgorithm: tdfConfig.segmentIntegrityAlgorithm, + segmentSize: tdfConfig.defaultSegmentSize, + useHex: tdfConfig.useHex, + }) +} + +// tdfFinalizeOptions maps the manifest-shaping parts of a TDFConfig onto the chunked +// writer's Finalize options. Encrypted metadata is absent here on purpose: the chunked +// writer only consults it when a KeySplitter builds the key access objects at Finalize +// time, and this path resolved them up front with the metadata already applied. +func tdfFinalizeOptions(tdfConfig *TDFConfig) ([]ChunkedFinalizeOption, error) { + assertions := slices.Clone(tdfConfig.assertions) if tdfConfig.addDefaultAssertion { systemMeta, err := GetSystemMetadataAssertionConfig() if err != nil { return nil, err } - tdfConfig.assertions = append(tdfConfig.assertions, systemMeta) + assertions = append(assertions, systemMeta) } - signedAssertion, err := signAssertions( - []byte(aggregateHashBuilder.String()), - tdfConfig.assertions, - tdfObject.payloadKey[:], - tdfConfig.useHex, - ) - if err != nil { - return nil, err - } - - tdfObject.manifest.Assertions = signedAssertion - - manifestAsStr, err := json.Marshal(tdfObject.manifest) - if err != nil { - return nil, fmt.Errorf("json.Marshal failed:%w", err) - } - - finalBytes, err := archiveWriter.Finalize(ctx, manifestAsStr) - if err != nil { - return nil, fmt.Errorf("zipstream.Finalize failed: %w", err) + opts := []ChunkedFinalizeOption{WithChunkedAssertions(assertions)} + if tdfConfig.mimeType != "" { + opts = append(opts, WithChunkedMimeType(tdfConfig.mimeType)) } - - _, err = outputWriter.Write(finalBytes) - if err != nil { - return nil, fmt.Errorf("io.writer.Write failed: %w", err) - } - - if err := archiveWriter.Close(); err != nil { - return nil, fmt.Errorf("zipstream.Close failed: %w", err) + if tdfConfig.excludeVersionFromManifest { + opts = append(opts, WithChunkedExcludeVersion()) } - - tdfObject.size = outputWriter.written - - return tdfObject, nil + return opts, nil } // resolveInputSize reports the payload length in bytes, or inputSizeUnknown when it @@ -527,37 +495,34 @@ func (r *Reader) Manifest() Manifest { return r.manifest } -// prepare the manifest for TDF -func (s SDK) prepareManifest(ctx context.Context, t *TDFObject, tdfConfig TDFConfig) error { //nolint:funlen,gocognit // Better readability keeping it as is - manifest := Manifest{} - - if !tdfConfig.excludeVersionFromManifest { - manifest.TDFVersion = TDFSpecVersion - } - - if len(tdfConfig.kaoTemplate) == 0 { - return fmt.Errorf("no key access template specified or inferred in initKAOTemplate: %w", errInvalidKasInfo) - } - - manifest.KeyAccessType = kSplitKeyType - - policyObj, err := createPolicyObject(tdfConfig.attributes) +// resolveKeyAccess builds the two manifest fields that bind the DEK to policy for the +// classic CreateTDF path: the base64-encoded policy object and the key access objects. +// The KAO template says which KAS servers hold which split; the DEK is divided among +// those splits and each share wrapped to the KAS servers assigned to it. +func (s SDK) resolveKeyAccess(ctx context.Context, tdfConfig *TDFConfig, dek []byte) (string, []KeyAccess, error) { + shares, err := s.templateSplitShares(ctx, tdfConfig, dek) if err != nil { - return fmt.Errorf("fail to create policy object:%w", err) + return "", nil, err } - policyObjectAsStr, err := json.Marshal(policyObj) - if err != nil { - return fmt.Errorf("json.Marshal failed:%w", err) + fqns := make([]string, 0, len(tdfConfig.attributes)) + for _, attribute := range tdfConfig.attributes { + fqns = append(fqns, attribute.String()) } + return resolvePolicyAndKeyAccess(fqns, shares, tdfConfig.metaData) +} - base64PolicyObject := ocrypto.Base64Encode(policyObjectAsStr) - - conjunction := make(map[string][]KASInfo) - var splitIDs []string +// templateSplitShares groups the KAO template by split ID and divides the DEK across +// the resulting shares. KAS entries sharing a split ID form an OR-group: any one of +// them can unwrap that share. Public keys absent from the template are fetched here. +func (s SDK) templateSplitShares(ctx context.Context, tdfConfig *TDFConfig, dek []byte) ([]splitShare, error) { + if len(tdfConfig.kaoTemplate) == 0 { + return nil, fmt.Errorf("no key access template specified or inferred in initKAOTemplate: %w", errInvalidKasInfo) + } + var shares []splitShare + index := map[string]int{} for _, tpl := range tdfConfig.kaoTemplate { - // Public key was passed in with kasInfoList ki := KASInfo{ URL: tpl.KAS, KID: tpl.kid, @@ -571,71 +536,29 @@ func (s SDK) prepareManifest(ctx context.Context, t *TDFObject, tdfConfig TDFCon } k, err := s.getPublicKey(ctx, tpl.KAS, a, tpl.kid) if err != nil { - return fmt.Errorf("unable to retrieve public key from KAS at [%s]: %w", tpl.KAS, err) + return nil, fmt.Errorf("unable to retrieve public key from KAS at [%s]: %w", tpl.KAS, err) } ki = *k } - if _, ok := conjunction[tpl.SplitID]; ok { - conjunction[tpl.SplitID] = append(conjunction[tpl.SplitID], ki) - } else { - conjunction[tpl.SplitID] = []KASInfo{ki} - splitIDs = append(splitIDs, tpl.SplitID) - } - } - - symKeys := make([][]byte, 0, len(splitIDs)) - for _, splitID := range splitIDs { - symKey, err := ocrypto.RandomBytes(kKeySize) - if err != nil { - return fmt.Errorf("ocrypto.RandomBytes failed:%w", err) - } - symKeys = append(symKeys, symKey) - - // policy binding - policyBinding := createPolicyBinding(symKey, base64PolicyObject) - - // encrypted metadata - // add meta data - var encryptedMetadata string - if len(tdfConfig.metaData) > 0 { - encryptedMetadata, err = encryptMetadata(symKey, tdfConfig.metaData) - if err != nil { - return err - } - } - - for _, kasInfo := range conjunction[splitID] { - if len(kasInfo.PublicKey) == 0 { - return fmt.Errorf("splitID:[%s], kas:[%s]: %w", splitID, kasInfo.URL, errKasPubKeyMissing) - } - - keyAccess, err := createKeyAccess(kasInfo, symKey, policyBinding, encryptedMetadata, splitID) - if err != nil { - return err - } - - manifest.KeyAccessObjs = append(manifest.KeyAccessObjs, keyAccess) - } - } - - manifest.Policy = string(base64PolicyObject) - manifest.Method.Algorithm = kGCMCipherAlgorithm - - // create the payload key by XOR all the keys in key access object. - for _, symKey := range symKeys { - for keyByteIndex, keyByte := range symKey { - t.payloadKey[keyByteIndex] ^= keyByte + if i, ok := index[tpl.SplitID]; ok { + shares[i].kases = append(shares[i].kases, ki) + continue } + index[tpl.SplitID] = len(shares) + shares = append(shares, splitShare{id: tpl.SplitID, kases: []KASInfo{ki}}) } - gcm, err := ocrypto.NewAESGcm(t.payloadKey[:]) + // The shares XOR back to the DEK, so every one of them is needed to reconstruct + // it — that is the AND half of the split semantics, with the OR half expressed by + // the several KAS servers that may sit on a single share. + data, err := splitDEK(dek, len(shares), defaultRand) if err != nil { - return fmt.Errorf(" ocrypto.NewAESGcm failed:%w", err) + return nil, err } - - t.manifest = manifest - t.aesGcm = gcm - return nil + for i := range shares { + shares[i].data = data[i] + } + return shares, nil } // integrityAlgorithmString maps an IntegrityAlgorithm to its manifest @@ -653,10 +576,11 @@ func integrityAlgorithmString(a IntegrityAlgorithm) string { return gmacIntegrityAlgorithm } -// createPolicyBinding produces an HMAC-SHA256 binding value keyed on the -// symmetric key, over the base64-encoded policy object. -func createPolicyBinding(symKey []byte, base64PolicyObject []byte) PolicyBinding { - policyBindingHash := hex.EncodeToString(ocrypto.CalculateSHA256Hmac(symKey, base64PolicyObject)) +// createPolicyBinding binds a key split to the policy it unlocks, keyed on the split +// itself so that a KAS can verify the policy it is asked to enforce is the one the +// creator bound. +func createPolicyBinding(symKey []byte, base64Policy string) PolicyBinding { + policyBindingHash := hex.EncodeToString(ocrypto.CalculateSHA256Hmac(symKey, []byte(base64Policy))) return PolicyBinding{ Alg: hmacIntegrityAlgorithm, Hash: string(ocrypto.Base64Encode([]byte(policyBindingHash))), @@ -810,8 +734,8 @@ func generateWrapKeyWithKEM(ktype ocrypto.KeyType, publicKeyPEM string, symKey [ return string(ocrypto.Base64Encode(wrappedDER)), scheme, nil } -// create policy object -func createPolicyObject(attributes []AttributeValueFQN) (PolicyObject, error) { +// createPolicyObjectFromFQNs builds the TDF policy document from attribute value FQNs. +func createPolicyObjectFromFQNs(fqns []string) (PolicyObject, error) { uuidObj, err := uuid.NewUUID() if err != nil { return PolicyObject{}, fmt.Errorf("uuid.NewUUID failed: %w", err) @@ -820,9 +744,9 @@ func createPolicyObject(attributes []AttributeValueFQN) (PolicyObject, error) { policyObj := PolicyObject{} policyObj.UUID = uuidObj.String() - for _, attribute := range attributes { + for _, fqn := range fqns { attributeObj := attributeObject{} - attributeObj.Attribute = attribute.String() + attributeObj.Attribute = fqn policyObj.Body.DataAttributes = append(policyObj.Body.DataAttributes, attributeObj) policyObj.Body.Dissem = make([]string, 0) } diff --git a/sdk/tdf_helpers_test.go b/sdk/tdf_helpers_test.go index eaf9a5095f..4dfab3e88d 100644 --- a/sdk/tdf_helpers_test.go +++ b/sdk/tdf_helpers_test.go @@ -59,7 +59,7 @@ func TestCreatePolicyBinding(t *testing.T) { fixedKey[i] = byte(i) } - binding := createPolicyBinding(fixedKey, ocrypto.Base64Encode([]byte(`{"uuid":"test"}`))) + binding := createPolicyBinding(fixedKey, string(ocrypto.Base64Encode([]byte(`{"uuid":"test"}`)))) assert.Equal(t, "YzFjZTM3OWQ0Y2FiMTZkNmRhNzJkYjllYWQ2NGQ3Y2I0Y2E5YmRhY2FiOGMwNjg1ZmY5MmUzZjc0YWEyYzEyZA==", @@ -67,7 +67,7 @@ func TestCreatePolicyBinding(t *testing.T) { }) t.Run("binds with HS256 over base64 policy", func(t *testing.T) { - binding := createPolicyBinding(symKey, ocrypto.Base64Encode([]byte(policyJSON))) + binding := createPolicyBinding(symKey, string(ocrypto.Base64Encode([]byte(policyJSON)))) assert.Equal(t, hmacIntegrityAlgorithm, binding.Alg) require.NotEmpty(t, binding.Hash) @@ -76,8 +76,8 @@ func TestCreatePolicyBinding(t *testing.T) { }) t.Run("different policies bind differently", func(t *testing.T) { - b1 := createPolicyBinding(symKey, ocrypto.Base64Encode([]byte(`{"policy":"test1"}`))) - b2 := createPolicyBinding(symKey, ocrypto.Base64Encode([]byte(`{"policy":"test2"}`))) + b1 := createPolicyBinding(symKey, string(ocrypto.Base64Encode([]byte(`{"policy":"test1"}`)))) + b2 := createPolicyBinding(symKey, string(ocrypto.Base64Encode([]byte(`{"policy":"test2"}`)))) assert.NotEqual(t, b1.Hash, b2.Hash) }) @@ -86,7 +86,7 @@ func TestCreatePolicyBinding(t *testing.T) { _, err := rand.Read(otherKey) require.NoError(t, err) - policy := ocrypto.Base64Encode([]byte(policyJSON)) + policy := string(ocrypto.Base64Encode([]byte(policyJSON))) assert.NotEqual(t, createPolicyBinding(symKey, policy).Hash, createPolicyBinding(otherKey, policy).Hash,