diff --git a/sdk/chunked_test.go b/sdk/chunked_test.go index 4079337a8d..285431e7a8 100644 --- a/sdk/chunked_test.go +++ b/sdk/chunked_test.go @@ -753,19 +753,17 @@ func TestChunkedECKeyAccess(t *testing.T) { for i := range dek { dek[i] = byte(i) } - splits := &SplitResult{ - KASPublicKeys: map[string]KASPublicKey{ - kasURL: { - Algorithm: string(ocrypto.EC256Key), - KID: "ec-kid", - PEM: pubPEM, - URL: kasURL, - }, - }, - Splits: []Split{{Data: dek, KASURLs: []string{kasURL}}}, - } + shares := []splitShare{{ + data: dek, + kases: []KASInfo{{ + URL: kasURL, + PublicKey: pubPEM, + KID: "ec-kid", + Algorithm: string(ocrypto.EC256Key), + }}, + }} - kaos, err := buildChunkedKeyAccessObjects(splits, []byte(`{"uuid":"test"}`), "") + kaos, err := buildKeyAccessObjects(shares, `{"uuid":"test"}`, "") require.NoError(t, err) require.Len(t, kaos, 1) diff --git a/sdk/chunked_writer.go b/sdk/chunked_writer.go index 79678a790d..95b81067e1 100644 --- a/sdk/chunked_writer.go +++ b/sdk/chunked_writer.go @@ -14,7 +14,6 @@ import ( "sync" "time" - "github.com/google/uuid" "github.com/opentdf/platform/lib/ocrypto" "github.com/opentdf/platform/protocol/go/policy" "github.com/opentdf/platform/sdk/internal/zipstream" @@ -232,6 +231,11 @@ type ChunkedWriterConfig struct { // fixedClock for deterministic ZIP output. clock clock + // dek is a pre-generated Data Encryption Key. When nil the writer + // draws one from rand. SDK.CreateTDF presets it so that it can + // resolve key access before emitting any payload bytes. + dek []byte + // excludeVersion omits the schemaVersion field from the manifest. // Set together with useHex by WithChunkedTargetMode; readers use // the field's absence as the pre-4.3.0 marker, so the two must @@ -250,6 +254,10 @@ type ChunkedWriterConfig struct { // signature. Defaults to HS256. integrityAlgorithm IntegrityAlgorithm + // keyAccess resolves the manifest policy and key access objects. + // Defaults to a splitterKeyAccess over splitter. + keyAccess keyAccessResolver + // rand is the entropy source used to generate the DEK. Defaults // to crypto/rand.Reader. rand io.Reader @@ -258,8 +266,14 @@ type ChunkedWriterConfig struct { // integrity hashes. Defaults to HS256. segmentIntegrityAlgorithm IntegrityAlgorithm + // segmentSize is the plaintext segment size advertised in the + // manifest. Zero means "report the first segment's actual size", + // which is right when every segment is the same length. + segmentSize int64 + // splitter maps attribute values to DEK splits at Finalize time. - // Defaults to DefaultKeySplitter (single-KAS only). + // Defaults to DefaultKeySplitter (single-KAS only). Ignored when + // keyAccess is set. splitter KeySplitter // useHex hex-encodes segment, root, and assertion signatures @@ -352,6 +366,10 @@ type chunkedWriter struct { // integrityAlgorithm is used for the root signature. integrityAlgorithm IntegrityAlgorithm + // keyAccess resolves the manifest policy and key access objects + // for the DEK. + keyAccess keyAccessResolver + // manifest holds the finalized manifest for post-Finalize // GetManifest calls. manifest *Manifest @@ -365,9 +383,9 @@ type chunkedWriter struct { // segments records per-index Segment metadata (hash + sizes). segments map[int]*Segment - // splitter converts attributes + DEK into key splits at - // Finalize time. - splitter KeySplitter + // segmentSize is the plaintext segment size to advertise in the + // manifest, or zero to infer it from the first segment. + segmentSize int64 // useHex selects the pre-4.3.0 doubly-encoded signature form. // Read by WriteSegment, so it is fixed at construction rather @@ -401,15 +419,30 @@ func NewChunkedWriter(_ context.Context, opts ...ChunkedWriterOption) (ChunkedWr return nil, err } } + return newChunkedWriter(cfg) +} - dek := make([]byte, kKeySize) - if _, err := io.ReadFull(cfg.rand, dek); err != nil { - return nil, fmt.Errorf("generate DEK: %w", err) +// newChunkedWriter builds the writer from a fully-populated config. +// SDK.CreateTDF calls this directly with the unexported knobs its +// classic behavior needs — a preset DEK, key access resolved before +// the first payload byte, a fixed segment size — rather than going +// through the public option set. +func newChunkedWriter(cfg ChunkedWriterConfig) (*chunkedWriter, error) { + dek := cfg.dek + if dek == nil { + dek = make([]byte, kKeySize) + if _, err := io.ReadFull(cfg.rand, dek); err != nil { + return nil, fmt.Errorf("generate DEK: %w", err) + } } block, err := cfg.cipherFactory(dek) if err != nil { return nil, fmt.Errorf("build segment cipher: %w", err) } + keyAccess := cfg.keyAccess + if keyAccess == nil { + keyAccess = splitterKeyAccess{splitter: cfg.splitter} + } return &chunkedWriter{ archiveWriter: cfg.archiveFactory(cfg.clock), block: block, @@ -418,9 +451,10 @@ func NewChunkedWriter(_ context.Context, opts ...ChunkedWriterOption) (ChunkedWr initialAttributes: cfg.initialAttributes, initialDefaultKAS: cfg.initialDefaultKAS, integrityAlgorithm: cfg.integrityAlgorithm, + keyAccess: keyAccess, segmentIntegrityAlgorithm: cfg.segmentIntegrityAlgorithm, segments: make(map[int]*Segment), - splitter: cfg.splitter, + segmentSize: cfg.segmentSize, useHex: cfg.useHex, }, nil } @@ -716,18 +750,10 @@ func (w *chunkedWriter) snapshotLocked(keep []int) (*chunkedSnapshot, error) { // buildManifest assembles the manifest from a snapshot. It reads no // mutable writer state and holds no lock: every other field it touches -// (dek, splitter, the integrity algorithms, useHex) is fixed at +// (dek, keyAccess, the integrity algorithms, useHex) is fixed at // construction. func (w *chunkedWriter) buildManifest(ctx context.Context, cfg *ChunkedFinalizeConfig, snap *chunkedSnapshot) (*Manifest, int64, int64, error) { - splits, err := w.splitter.Split(ctx, cfg.attributes, w.dek, cfg.defaultKAS) - if err != nil { - return nil, 0, 0, err - } - policyBytes, err := buildChunkedPolicy(cfg.attributes) - if err != nil { - return nil, 0, 0, err - } - kaos, err := buildChunkedKeyAccessObjects(splits, policyBytes, cfg.encryptedMetadata) + base64Policy, kaos, err := w.keyAccess.resolve(ctx, w.dek, cfg) if err != nil { return nil, 0, 0, err } @@ -735,7 +761,7 @@ func (w *chunkedWriter) buildManifest(ctx context.Context, cfg *ChunkedFinalizeC encInfo := EncryptionInformation{ KeyAccessObjs: kaos, KeyAccessType: kSplitKeyType, - Policy: string(ocrypto.Base64Encode(policyBytes)), + Policy: base64Policy, Method: Method{ Algorithm: kGCMCipherAlgorithm, IsStreamable: true, @@ -758,7 +784,15 @@ func (w *chunkedWriter) buildManifest(ctx context.Context, cfg *ChunkedFinalizeC } aggregate.Write(decoded) } - if len(snap.segments) > 0 { + // A caller that knows the segment size says so, because the first + // segment's actual length is only the right answer when every + // segment is full — and the last one usually is not, so a + // single-segment TDF would otherwise advertise a short default. + switch { + case w.segmentSize > 0: + encInfo.DefaultSegmentSize = w.segmentSize + encInfo.DefaultEncryptedSegSize = w.segmentSize + gcmIvSize + aesBlockSize + case len(snap.segments) > 0: encInfo.DefaultEncryptedSegSize = snap.segments[0].EncryptedSize encInfo.DefaultSegmentSize = snap.segments[0].Size } @@ -852,66 +886,6 @@ func (w *chunkedWriter) segmentOrderLocked(keep []int) ([]int, error) { return out, nil } -// buildChunkedKeyAccessObjects wraps each split share to each KAS -// listed by the splitter. -func buildChunkedKeyAccessObjects(splits *SplitResult, policyBytes []byte, metadata string) ([]KeyAccess, error) { - if splits == nil || len(splits.Splits) == 0 { - return nil, errors.New("no splits produced") - } - base64Policy := ocrypto.Base64Encode(policyBytes) - - var out []KeyAccess - for _, split := range splits.Splits { - // Policy binding and metadata are keyed on the split share, not - // on the KAS, so compute them once per split rather than once - // per KAS URL in an OR-group. - policyBinding := createPolicyBinding(split.Data, base64Policy) - var encMeta string - if metadata != "" { - m, err := encryptMetadata(split.Data, metadata) - if err != nil { - return nil, fmt.Errorf("encrypt metadata for split %s: %w", split.ID, err) - } - encMeta = m - } - for _, url := range split.KASURLs { - // A KAS named by a split but absent from KASPublicKeys is an - // error, not something to skip. Dropping it silently removes - // the only KAO that would have let that KAS unwrap this - // share; if every URL on the split is missing, the share - // becomes unrecoverable and the TDF undecryptable, with - // nothing in the output to say why. - pk, ok := splits.KASPublicKeys[url] - if !ok || pk.PEM == "" { - return nil, fmt.Errorf("splitID:[%s], kas:[%s]: %w", split.ID, url, errKasPubKeyMissing) - } - kao, err := createKeyAccess(pk.toKASInfo(), split.Data, policyBinding, encMeta, split.ID) - if err != nil { - return nil, fmt.Errorf("wrap key for %s: %w", url, err) - } - out = append(out, kao) - } - } - if len(out) == 0 { - return nil, errors.New("no valid key access objects generated") - } - return out, nil -} - -// buildChunkedPolicy composes the TDF Policy document from attribute -// values. -func buildChunkedPolicy(values []*policy.Value) ([]byte, error) { - p := PolicyObject{UUID: uuid.NewString()} - p.Body.DataAttributes = make([]attributeObject, 0, len(values)) - p.Body.Dissem = make([]string, 0) - for _, v := range values { - p.Body.DataAttributes = append(p.Body.DataAttributes, attributeObject{ - Attribute: v.GetFqn(), - }) - } - return json.Marshal(p) -} - // cloneChunkedManifest returns a shallow-deep copy safe to hand out. func cloneChunkedManifest(in *Manifest) *Manifest { if in == nil { diff --git a/sdk/key_splitter.go b/sdk/key_splitter.go index a40fded1f3..470fea684a 100644 --- a/sdk/key_splitter.go +++ b/sdk/key_splitter.go @@ -2,9 +2,12 @@ package sdk import ( "context" + "encoding/json" "errors" "fmt" + "io" + "github.com/opentdf/platform/lib/ocrypto" "github.com/opentdf/platform/protocol/go/policy" ) @@ -73,18 +76,6 @@ type KASPublicKey struct { URL string } -// toKASInfo adapts the splitter's wrapping-key descriptor to the -// KASInfo shape consumed by createKeyAccess. Default is not carried -// over; it plays no part in building a key access object. -func (k KASPublicKey) toKASInfo() KASInfo { - return KASInfo{ - URL: k.URL, - PublicKey: k.PEM, - KID: k.KID, - Algorithm: k.Algorithm, - } -} - // ErrSplitterRequiresDefaultKAS is returned by the default key // splitter when no default KAS was supplied. The default splitter is // single-KAS only; multi-attribute splits require injecting a full @@ -155,6 +146,170 @@ func (s *singleKASSplitter) Split(_ context.Context, _ []*policy.Value, dek []by }, nil } +// keyAccessResolver turns a DEK into the two manifest fields that +// bind it to policy: the base64-encoded policy object and the key +// access objects wrapping the DEK to each KAS. The writer holds one; +// SDK.CreateTDF resolves its key access up front and supplies a +// staticKeyAccess, while the chunked path defers to a KeySplitter at +// Finalize time. +type keyAccessResolver interface { + resolve(ctx context.Context, dek []byte, cfg *ChunkedFinalizeConfig) (string, []KeyAccess, error) +} + +// splitShare is one XOR share of the DEK together with every KAS able +// to unwrap it. Several KAS entries on one share mean any of them +// suffices (OR semantics); several shares mean all are required (AND). +type splitShare struct { + // id names the share in the manifest ("sid"). Empty when the TDF + // has a single share. + id string + + // data is the share itself. + data []byte + + // kases are the wrapping targets for this share. + kases []KASInfo +} + +// staticKeyAccess returns key access objects resolved ahead of time. +type staticKeyAccess struct { + // kaos are the pre-built key access objects. + kaos []KeyAccess + + // policy is the base64-encoded policy object the kaos are bound to. + policy string +} + +func (r staticKeyAccess) resolve(_ context.Context, _ []byte, _ *ChunkedFinalizeConfig) (string, []KeyAccess, error) { + return r.policy, r.kaos, nil +} + +// splitterKeyAccess adapts a public KeySplitter to keyAccessResolver. +type splitterKeyAccess struct { + // splitter maps attributes plus the DEK onto KAS-addressed shares. + splitter KeySplitter +} + +func (r splitterKeyAccess) resolve(ctx context.Context, dek []byte, cfg *ChunkedFinalizeConfig) (string, []KeyAccess, error) { + splits, err := r.splitter.Split(ctx, cfg.attributes, dek, cfg.defaultKAS) + if err != nil { + return "", nil, err + } + if splits == nil || len(splits.Splits) == 0 { + return "", nil, errors.New("no splits produced") + } + + shares := make([]splitShare, 0, len(splits.Splits)) + for _, split := range splits.Splits { + share := splitShare{id: split.ID, data: split.Data} + for _, url := range split.KASURLs { + // A KAS named by a split but absent from KASPublicKeys + // yields an empty PEM here, which buildKeyAccessObjects + // rejects. Carrying it through rather than skipping it is + // deliberate: dropping it silently removes the only KAO + // that would have let that KAS unwrap this share, and if + // every URL on the split is missing, the share becomes + // unrecoverable and the TDF undecryptable with nothing in + // the output to say why. + pk := splits.KASPublicKeys[url] + share.kases = append(share.kases, KASInfo{ + URL: url, + PublicKey: pk.PEM, + KID: pk.KID, + Algorithm: pk.Algorithm, + }) + } + shares = append(shares, share) + } + + fqns := make([]string, 0, len(cfg.attributes)) + for _, v := range cfg.attributes { + fqns = append(fqns, v.GetFqn()) + } + return resolvePolicyAndKeyAccess(fqns, shares, cfg.encryptedMetadata) +} + +// resolvePolicyAndKeyAccess builds the policy document the DEK is bound to and wraps +// every share to its KAS targets, returning the two manifest fields that bind a DEK to +// policy. Shared by SDK.CreateTDF's KAO template path and the chunked writer's +// KeySplitter path, so both emit byte-identical policy for the same attributes. +func resolvePolicyAndKeyAccess(fqns []string, shares []splitShare, metadata string) (string, []KeyAccess, error) { + policyObj, err := createPolicyObjectFromFQNs(fqns) + if err != nil { + return "", nil, fmt.Errorf("fail to create policy object:%w", err) + } + policyObjectAsStr, err := json.Marshal(policyObj) + if err != nil { + return "", nil, fmt.Errorf("json.Marshal failed:%w", err) + } + base64Policy := string(ocrypto.Base64Encode(policyObjectAsStr)) + + kaos, err := buildKeyAccessObjects(shares, base64Policy, metadata) + if err != nil { + return "", nil, err + } + return base64Policy, kaos, nil +} + +// buildKeyAccessObjects wraps every share to each of its KAS targets, +// emitting the manifest's keyAccess array in share order. +func buildKeyAccessObjects(shares []splitShare, base64Policy, metadata string) ([]KeyAccess, error) { + var kaos []KeyAccess + for _, share := range shares { + // Policy binding and metadata are keyed on the split share, not + // on the KAS, so compute them once per share rather than once + // per KAS URL in an OR-group. + policyBinding := createPolicyBinding(share.data, base64Policy) + + var encryptedMetadata string + if metadata != "" { + var err error + encryptedMetadata, err = encryptMetadata(share.data, metadata) + if err != nil { + return nil, err + } + } + + for _, kasInfo := range share.kases { + if kasInfo.PublicKey == "" { + return nil, fmt.Errorf("splitID:[%s], kas:[%s]: %w", share.id, kasInfo.URL, errKasPubKeyMissing) + } + keyAccess, err := createKeyAccess(kasInfo, share.data, policyBinding, encryptedMetadata, share.id) + if err != nil { + return nil, err + } + kaos = append(kaos, keyAccess) + } + } + if len(kaos) == 0 { + return nil, errors.New("no key access objects generated") + } + return kaos, nil +} + +// splitDEK returns count XOR shares of dek. Every share but the last +// is random; the last absorbs the parity so the shares XOR back to dek. +func splitDEK(dek []byte, count int, rand io.Reader) ([][]byte, error) { + if count <= 0 { + return nil, errors.New("no key splits requested") + } + shares := make([][]byte, count) + parity := make([]byte, len(dek)) + copy(parity, dek) + for i := range count - 1 { + share := make([]byte, len(dek)) + if _, err := io.ReadFull(rand, share); err != nil { + return nil, fmt.Errorf("generate key split failed: %w", err) + } + for j, b := range share { + parity[j] ^= b + } + shares[i] = share + } + shares[count-1] = parity + return shares, nil +} + // algorithmPolicyToString maps a policy.Algorithm enum to the // ocrypto.KeyType string form used when picking a wrap scheme. // Unknown enums, including ALGORITHM_UNSPECIFIED, return the empty diff --git a/sdk/tdf.go b/sdk/tdf.go index 30db9f5ae9..f2efc3f0ae 100644 --- a/sdk/tdf.go +++ b/sdk/tdf.go @@ -8,10 +8,10 @@ import ( "encoding/json" "errors" "fmt" - "hash/crc32" "io" "log/slog" "net/http" + "slices" "strconv" "strings" @@ -76,10 +76,8 @@ type RequiredObligations struct { } type TDFObject struct { - manifest Manifest - size int64 - aesGcm ocrypto.AesGcm - payloadKey [kKeySize]byte + manifest Manifest + size int64 } type countingWriter struct { @@ -179,12 +177,6 @@ func (s SDK) CreateTDFContext(ctx context.Context, writer io.Writer, reader io.R return nil, err } - tdfObject := &TDFObject{} - err = s.prepareManifest(ctx, tdfObject, *tdfConfig) - if err != nil { - return nil, fmt.Errorf("fail to create a new split key: %w", err) - } - segmentSize := tdfConfig.defaultSegmentSize if segmentSize > maxSegmentSize { return nil, fmt.Errorf("segment size too large: %d", segmentSize) @@ -198,41 +190,27 @@ func (s SDK) CreateTDFContext(ctx context.Context, writer io.Writer, reader io.R } totalSegments := segmentCount(inputSize, segmentSize) - encryptedSegmentSize := segmentSize + gcmIvSize + aesBlockSize - - // The ZIP64 choice is baked into the payload's local file header, which goes out - // ahead of the first segment, so it cannot be revisited once the archive has - // started. Reserve ZIP64 for payloads that a 32-bit offset cannot address — and - // for payloads of unknown length, which might turn out to be one. - payloadSize := inputSize + int64(totalSegments)*(gcmIvSize+aesBlockSize) - zipMode := zipstream.Zip64Auto - if inputSize == inputSizeUnknown || payloadSize >= zip64MagicVal { - zipMode = zipstream.Zip64Always - } - - archiveOpts := []zipstream.Option{zipstream.WithZip64Mode(zipMode)} - if totalSegments > 0 { - archiveOpts = append(archiveOpts, zipstream.WithMaxSegments(totalSegments)) + chunked, err := s.newTDFChunkedWriter(ctx, tdfConfig, inputSize, totalSegments) + if err != nil { + return nil, err } - archiveWriter := zipstream.NewSegmentTDFWriter(totalSegments, archiveOpts...) outputWriter := &countingWriter{writer: writer} // A known length doubles as a read limit: overrunning it would invalidate the - // ZIP64 choice made from it above. Only as large as the payload actually needs, - // too — the segment size defaults to 2 MiB, so sizing on it alone would allocate - // that much to encrypt a handful of bytes. The buffer never shrinks to zero, so a - // read that comes back empty always means EOF. + // ZIP64 choice made from it in newTDFChunkedWriter. Only as large as the payload + // actually needs, too — the segment size defaults to 2 MiB, so sizing on it alone + // would allocate that much to encrypt a handful of bytes. The buffer never shrinks + // to zero, so a read that comes back empty always means EOF. readBufSize := segmentSize if inputSize != inputSizeUnknown { reader = io.LimitReader(reader, inputSize) readBufSize = max(1, min(segmentSize, inputSize)) } - var aggregateHashBuilder strings.Builder var bytesRead int64 readBuf := make([]byte, readBufSize) - for segmentIndex := 0; ; segmentIndex++ { + for index := 0; ; index++ { // io.Reader.Read is free to return fewer bytes than asked for without // erroring, so a bare Read would cut segments short at the whim of the // reader. ReadFull retries until the segment is filled or the input runs @@ -243,50 +221,19 @@ func (s SDK) CreateTDFContext(ctx context.Context, writer io.Writer, reader io.R } // A payload whose length is an exact multiple of the segment size reports // EOF with nothing read. An empty payload still gets one empty segment. - if n == 0 && segmentIndex > 0 { + if n == 0 && index > 0 { break } - readSize := int64(n) - bytesRead += readSize + bytesRead += int64(n) - cipherData, err := tdfObject.aesGcm.Encrypt(readBuf[:readSize]) + segment, err := chunked.WriteSegment(ctx, index, readBuf[:n]) if err != nil { - return nil, fmt.Errorf("ocrypto.AesGcm.Encrypt failed: %w", err) - } - - crc := crc32.ChecksumIEEE(cipherData) - headerBytes, err := archiveWriter.WriteSegment(ctx, segmentIndex, uint64(len(cipherData)), crc) - if err != nil { - return nil, fmt.Errorf("zipstream.WriteSegment failed: %w", err) - } - - if len(headerBytes) > 0 { - _, err = outputWriter.Write(headerBytes) - if err != nil { - return nil, fmt.Errorf("io.writer.Write failed: %w", err) - } + return nil, err } - - _, err = outputWriter.Write(cipherData) - if err != nil { + if _, err := io.Copy(outputWriter, segment.TDFData); err != nil { return nil, fmt.Errorf("io.writer.Write failed: %w", err) } - segmentSig, err := calculateSignature(cipherData, tdfObject.payloadKey[:], - tdfConfig.segmentIntegrityAlgorithm, tdfConfig.useHex) - if err != nil { - return nil, fmt.Errorf("splitKey.GetSignaturefailed: %w", err) - } - - aggregateHashBuilder.WriteString(segmentSig) - segmentInfo := Segment{ - Hash: string(ocrypto.Base64Encode([]byte(segmentSig))), - Size: readSize, - EncryptedSize: int64(len(cipherData)), - } - - tdfObject.manifest.Segments = append(tdfObject.manifest.Segments, segmentInfo) - if readErr != nil { break } @@ -300,76 +247,97 @@ func (s SDK) CreateTDFContext(ctx context.Context, writer io.Writer, reader io.R return nil, fmt.Errorf("%w: read %d of %d bytes", errInputShorterThanDeclared, bytesRead, inputSize) } - rootSignature, err := calculateSignature([]byte(aggregateHashBuilder.String()), tdfObject.payloadKey[:], - tdfConfig.integrityAlgorithm, tdfConfig.useHex) + finalizeOpts, err := tdfFinalizeOptions(tdfConfig) if err != nil { - return nil, fmt.Errorf("splitKey.GetSignaturefailed: %w", err) + return nil, err } - sig := string(ocrypto.Base64Encode([]byte(rootSignature))) - tdfObject.manifest.Signature = sig + result, err := chunked.Finalize(ctx, finalizeOpts...) + if err != nil { + return nil, err + } - tdfObject.manifest.Algorithm = integrityAlgorithmString(tdfConfig.integrityAlgorithm) + if _, err := outputWriter.Write(result.Data); err != nil { + return nil, fmt.Errorf("io.writer.Write failed: %w", err) + } - tdfObject.manifest.DefaultSegmentSize = segmentSize - tdfObject.manifest.DefaultEncryptedSegSize = encryptedSegmentSize + return &TDFObject{ + manifest: *result.Manifest, + size: outputWriter.written, + }, nil +} - tdfObject.manifest.SegmentHashAlgorithm = integrityAlgorithmString(tdfConfig.segmentIntegrityAlgorithm) - tdfObject.manifest.Method.IsStreamable = true +// newTDFChunkedWriter builds the chunked writer backing SDK.CreateTDF. Key access is +// resolved here, before any payload byte is written, so that an unreachable KAS fails +// the call without leaving a partial TDF on the output writer. +// +// inputSize may be inputSizeUnknown and totalSegments zero, for a payload that can only +// be measured by reading it. +func (s SDK) newTDFChunkedWriter(ctx context.Context, tdfConfig *TDFConfig, inputSize int64, totalSegments int) (*chunkedWriter, error) { + dek := make([]byte, kKeySize) + if _, err := io.ReadFull(defaultRand, dek); err != nil { + return nil, fmt.Errorf("fail to create a new split key: %w", err) + } - // add payload info - mimeType := tdfConfig.mimeType - if mimeType == "" { - mimeType = defaultMimeType + base64Policy, kaos, err := s.resolveKeyAccess(ctx, tdfConfig, dek) + if err != nil { + return nil, fmt.Errorf("fail to create a new split key: %w", err) } - tdfObject.manifest.MimeType = mimeType - tdfObject.manifest.Protocol = tdfAsZip - tdfObject.manifest.Type = tdfZipReference - tdfObject.manifest.URL = zipstream.TDFPayloadFileName - tdfObject.manifest.IsEncrypted = true + // The ZIP64 choice is baked into the payload's local file header, which goes out + // with segment 0, so it cannot be revisited once the archive has started. Reserve + // ZIP64 for payloads that a 32-bit offset cannot address — and for payloads of + // unknown length, which might turn out to be one. + payloadSize := inputSize + int64(totalSegments)*(gcmIvSize+aesBlockSize) + zipMode := zipstream.Zip64Auto + if inputSize == inputSizeUnknown || payloadSize >= zip64MagicVal { + zipMode = zipstream.Zip64Always + } + + return newChunkedWriter(ChunkedWriterConfig{ + archiveFactory: func(c clock) zipstream.SegmentWriter { + archiveOpts := []zipstream.Option{ + zipstream.WithZip64Mode(zipMode), + zipstream.WithClock(c.Now), + } + if totalSegments > 0 { + archiveOpts = append(archiveOpts, zipstream.WithMaxSegments(totalSegments)) + } + return zipstream.NewSegmentTDFWriter(totalSegments, archiveOpts...) + }, + cipherFactory: defaultSegmentCipherFactory, + clock: systemClock{}, + dek: dek, + integrityAlgorithm: tdfConfig.integrityAlgorithm, + keyAccess: staticKeyAccess{kaos: kaos, policy: base64Policy}, + segmentIntegrityAlgorithm: tdfConfig.segmentIntegrityAlgorithm, + segmentSize: tdfConfig.defaultSegmentSize, + useHex: tdfConfig.useHex, + }) +} + +// tdfFinalizeOptions maps the manifest-shaping parts of a TDFConfig onto the chunked +// writer's Finalize options. Encrypted metadata is absent here on purpose: the chunked +// writer only consults it when a KeySplitter builds the key access objects at Finalize +// time, and this path resolved them up front with the metadata already applied. +func tdfFinalizeOptions(tdfConfig *TDFConfig) ([]ChunkedFinalizeOption, error) { + assertions := slices.Clone(tdfConfig.assertions) if tdfConfig.addDefaultAssertion { systemMeta, err := GetSystemMetadataAssertionConfig() if err != nil { return nil, err } - tdfConfig.assertions = append(tdfConfig.assertions, systemMeta) + assertions = append(assertions, systemMeta) } - signedAssertion, err := signAssertions( - []byte(aggregateHashBuilder.String()), - tdfConfig.assertions, - tdfObject.payloadKey[:], - tdfConfig.useHex, - ) - if err != nil { - return nil, err - } - - tdfObject.manifest.Assertions = signedAssertion - - manifestAsStr, err := json.Marshal(tdfObject.manifest) - if err != nil { - return nil, fmt.Errorf("json.Marshal failed:%w", err) - } - - finalBytes, err := archiveWriter.Finalize(ctx, manifestAsStr) - if err != nil { - return nil, fmt.Errorf("zipstream.Finalize failed: %w", err) + opts := []ChunkedFinalizeOption{WithChunkedAssertions(assertions)} + if tdfConfig.mimeType != "" { + opts = append(opts, WithChunkedMimeType(tdfConfig.mimeType)) } - - _, err = outputWriter.Write(finalBytes) - if err != nil { - return nil, fmt.Errorf("io.writer.Write failed: %w", err) - } - - if err := archiveWriter.Close(); err != nil { - return nil, fmt.Errorf("zipstream.Close failed: %w", err) + if tdfConfig.excludeVersionFromManifest { + opts = append(opts, WithChunkedExcludeVersion()) } - - tdfObject.size = outputWriter.written - - return tdfObject, nil + return opts, nil } // resolveInputSize reports the payload length in bytes, or inputSizeUnknown when it @@ -527,37 +495,34 @@ func (r *Reader) Manifest() Manifest { return r.manifest } -// prepare the manifest for TDF -func (s SDK) prepareManifest(ctx context.Context, t *TDFObject, tdfConfig TDFConfig) error { //nolint:funlen,gocognit // Better readability keeping it as is - manifest := Manifest{} - - if !tdfConfig.excludeVersionFromManifest { - manifest.TDFVersion = TDFSpecVersion - } - - if len(tdfConfig.kaoTemplate) == 0 { - return fmt.Errorf("no key access template specified or inferred in initKAOTemplate: %w", errInvalidKasInfo) - } - - manifest.KeyAccessType = kSplitKeyType - - policyObj, err := createPolicyObject(tdfConfig.attributes) +// resolveKeyAccess builds the two manifest fields that bind the DEK to policy for the +// classic CreateTDF path: the base64-encoded policy object and the key access objects. +// The KAO template says which KAS servers hold which split; the DEK is divided among +// those splits and each share wrapped to the KAS servers assigned to it. +func (s SDK) resolveKeyAccess(ctx context.Context, tdfConfig *TDFConfig, dek []byte) (string, []KeyAccess, error) { + shares, err := s.templateSplitShares(ctx, tdfConfig, dek) if err != nil { - return fmt.Errorf("fail to create policy object:%w", err) + return "", nil, err } - policyObjectAsStr, err := json.Marshal(policyObj) - if err != nil { - return fmt.Errorf("json.Marshal failed:%w", err) + fqns := make([]string, 0, len(tdfConfig.attributes)) + for _, attribute := range tdfConfig.attributes { + fqns = append(fqns, attribute.String()) } + return resolvePolicyAndKeyAccess(fqns, shares, tdfConfig.metaData) +} - base64PolicyObject := ocrypto.Base64Encode(policyObjectAsStr) - - conjunction := make(map[string][]KASInfo) - var splitIDs []string +// templateSplitShares groups the KAO template by split ID and divides the DEK across +// the resulting shares. KAS entries sharing a split ID form an OR-group: any one of +// them can unwrap that share. Public keys absent from the template are fetched here. +func (s SDK) templateSplitShares(ctx context.Context, tdfConfig *TDFConfig, dek []byte) ([]splitShare, error) { + if len(tdfConfig.kaoTemplate) == 0 { + return nil, fmt.Errorf("no key access template specified or inferred in initKAOTemplate: %w", errInvalidKasInfo) + } + var shares []splitShare + index := map[string]int{} for _, tpl := range tdfConfig.kaoTemplate { - // Public key was passed in with kasInfoList ki := KASInfo{ URL: tpl.KAS, KID: tpl.kid, @@ -571,71 +536,29 @@ func (s SDK) prepareManifest(ctx context.Context, t *TDFObject, tdfConfig TDFCon } k, err := s.getPublicKey(ctx, tpl.KAS, a, tpl.kid) if err != nil { - return fmt.Errorf("unable to retrieve public key from KAS at [%s]: %w", tpl.KAS, err) + return nil, fmt.Errorf("unable to retrieve public key from KAS at [%s]: %w", tpl.KAS, err) } ki = *k } - if _, ok := conjunction[tpl.SplitID]; ok { - conjunction[tpl.SplitID] = append(conjunction[tpl.SplitID], ki) - } else { - conjunction[tpl.SplitID] = []KASInfo{ki} - splitIDs = append(splitIDs, tpl.SplitID) - } - } - - symKeys := make([][]byte, 0, len(splitIDs)) - for _, splitID := range splitIDs { - symKey, err := ocrypto.RandomBytes(kKeySize) - if err != nil { - return fmt.Errorf("ocrypto.RandomBytes failed:%w", err) - } - symKeys = append(symKeys, symKey) - - // policy binding - policyBinding := createPolicyBinding(symKey, base64PolicyObject) - - // encrypted metadata - // add meta data - var encryptedMetadata string - if len(tdfConfig.metaData) > 0 { - encryptedMetadata, err = encryptMetadata(symKey, tdfConfig.metaData) - if err != nil { - return err - } - } - - for _, kasInfo := range conjunction[splitID] { - if len(kasInfo.PublicKey) == 0 { - return fmt.Errorf("splitID:[%s], kas:[%s]: %w", splitID, kasInfo.URL, errKasPubKeyMissing) - } - - keyAccess, err := createKeyAccess(kasInfo, symKey, policyBinding, encryptedMetadata, splitID) - if err != nil { - return err - } - - manifest.KeyAccessObjs = append(manifest.KeyAccessObjs, keyAccess) - } - } - - manifest.Policy = string(base64PolicyObject) - manifest.Method.Algorithm = kGCMCipherAlgorithm - - // create the payload key by XOR all the keys in key access object. - for _, symKey := range symKeys { - for keyByteIndex, keyByte := range symKey { - t.payloadKey[keyByteIndex] ^= keyByte + if i, ok := index[tpl.SplitID]; ok { + shares[i].kases = append(shares[i].kases, ki) + continue } + index[tpl.SplitID] = len(shares) + shares = append(shares, splitShare{id: tpl.SplitID, kases: []KASInfo{ki}}) } - gcm, err := ocrypto.NewAESGcm(t.payloadKey[:]) + // The shares XOR back to the DEK, so every one of them is needed to reconstruct + // it — that is the AND half of the split semantics, with the OR half expressed by + // the several KAS servers that may sit on a single share. + data, err := splitDEK(dek, len(shares), defaultRand) if err != nil { - return fmt.Errorf(" ocrypto.NewAESGcm failed:%w", err) + return nil, err } - - t.manifest = manifest - t.aesGcm = gcm - return nil + for i := range shares { + shares[i].data = data[i] + } + return shares, nil } // integrityAlgorithmString maps an IntegrityAlgorithm to its manifest @@ -653,10 +576,11 @@ func integrityAlgorithmString(a IntegrityAlgorithm) string { return gmacIntegrityAlgorithm } -// createPolicyBinding produces an HMAC-SHA256 binding value keyed on the -// symmetric key, over the base64-encoded policy object. -func createPolicyBinding(symKey []byte, base64PolicyObject []byte) PolicyBinding { - policyBindingHash := hex.EncodeToString(ocrypto.CalculateSHA256Hmac(symKey, base64PolicyObject)) +// createPolicyBinding binds a key split to the policy it unlocks, keyed on the split +// itself so that a KAS can verify the policy it is asked to enforce is the one the +// creator bound. +func createPolicyBinding(symKey []byte, base64Policy string) PolicyBinding { + policyBindingHash := hex.EncodeToString(ocrypto.CalculateSHA256Hmac(symKey, []byte(base64Policy))) return PolicyBinding{ Alg: hmacIntegrityAlgorithm, Hash: string(ocrypto.Base64Encode([]byte(policyBindingHash))), @@ -810,8 +734,8 @@ func generateWrapKeyWithKEM(ktype ocrypto.KeyType, publicKeyPEM string, symKey [ return string(ocrypto.Base64Encode(wrappedDER)), scheme, nil } -// create policy object -func createPolicyObject(attributes []AttributeValueFQN) (PolicyObject, error) { +// createPolicyObjectFromFQNs builds the TDF policy document from attribute value FQNs. +func createPolicyObjectFromFQNs(fqns []string) (PolicyObject, error) { uuidObj, err := uuid.NewUUID() if err != nil { return PolicyObject{}, fmt.Errorf("uuid.NewUUID failed: %w", err) @@ -820,9 +744,9 @@ func createPolicyObject(attributes []AttributeValueFQN) (PolicyObject, error) { policyObj := PolicyObject{} policyObj.UUID = uuidObj.String() - for _, attribute := range attributes { + for _, fqn := range fqns { attributeObj := attributeObject{} - attributeObj.Attribute = attribute.String() + attributeObj.Attribute = fqn policyObj.Body.DataAttributes = append(policyObj.Body.DataAttributes, attributeObj) policyObj.Body.Dissem = make([]string, 0) } diff --git a/sdk/tdf_helpers_test.go b/sdk/tdf_helpers_test.go index eaf9a5095f..4dfab3e88d 100644 --- a/sdk/tdf_helpers_test.go +++ b/sdk/tdf_helpers_test.go @@ -59,7 +59,7 @@ func TestCreatePolicyBinding(t *testing.T) { fixedKey[i] = byte(i) } - binding := createPolicyBinding(fixedKey, ocrypto.Base64Encode([]byte(`{"uuid":"test"}`))) + binding := createPolicyBinding(fixedKey, string(ocrypto.Base64Encode([]byte(`{"uuid":"test"}`)))) assert.Equal(t, "YzFjZTM3OWQ0Y2FiMTZkNmRhNzJkYjllYWQ2NGQ3Y2I0Y2E5YmRhY2FiOGMwNjg1ZmY5MmUzZjc0YWEyYzEyZA==", @@ -67,7 +67,7 @@ func TestCreatePolicyBinding(t *testing.T) { }) t.Run("binds with HS256 over base64 policy", func(t *testing.T) { - binding := createPolicyBinding(symKey, ocrypto.Base64Encode([]byte(policyJSON))) + binding := createPolicyBinding(symKey, string(ocrypto.Base64Encode([]byte(policyJSON)))) assert.Equal(t, hmacIntegrityAlgorithm, binding.Alg) require.NotEmpty(t, binding.Hash) @@ -76,8 +76,8 @@ func TestCreatePolicyBinding(t *testing.T) { }) t.Run("different policies bind differently", func(t *testing.T) { - b1 := createPolicyBinding(symKey, ocrypto.Base64Encode([]byte(`{"policy":"test1"}`))) - b2 := createPolicyBinding(symKey, ocrypto.Base64Encode([]byte(`{"policy":"test2"}`))) + b1 := createPolicyBinding(symKey, string(ocrypto.Base64Encode([]byte(`{"policy":"test1"}`)))) + b2 := createPolicyBinding(symKey, string(ocrypto.Base64Encode([]byte(`{"policy":"test2"}`)))) assert.NotEqual(t, b1.Hash, b2.Hash) }) @@ -86,7 +86,7 @@ func TestCreatePolicyBinding(t *testing.T) { _, err := rand.Read(otherKey) require.NoError(t, err) - policy := ocrypto.Base64Encode([]byte(policyJSON)) + policy := string(ocrypto.Base64Encode([]byte(policyJSON))) assert.NotEqual(t, createPolicyBinding(symKey, policy).Hash, createPolicyBinding(otherKey, policy).Hash,