Skip to content

feat(sdk): graduate the chunked writer to stable API #3949

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dmihalcik-virtru wants to merge 1 commit into
base: dspx-2604-19-drop-encrypt-spool
Choose a base branch
from
Open

feat(sdk): graduate the chunked writer to stable API #3949

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 35 additions & 23 deletions examples/cmd/benchmark_experimental.go → examples/cmd/benchmark_chunked.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
"github.com/opentdf/platform/protocol/go/kas/kasconnect"
"github.com/opentdf/platform/protocol/go/policy"

"github.com/opentdf/platform/sdk/experimental/tdf"
"github.com/opentdf/platform/sdk"
"github.com/opentdf/platform/sdk/httputil"
"github.com/spf13/cobra"
)
Expand All @@ -27,10 +27,10 @@ var (

func init() {
benchmarkCmd := &cobra.Command{
Use: "benchmark-experimental-writer",
Short: "Benchmark experimental TDF writer speed",
Long: `Benchmark the experimental TDF writer with configurable payload size.`,
RunE: runExperimentalWriterBenchmark,
Use: "benchmark-chunked-writer",
Short: "Benchmark chunked TDF writer speed",
Long: `Benchmark the chunked TDF writer with configurable payload size.`,
RunE: runChunkedWriterBenchmark,
}
//nolint: mnd // no magic number, this is just default value for payload size
benchmarkCmd.Flags().IntVar(&payloadSize, "payload-size", 1024*1024, "Payload size in bytes") // Default 1MB
Expand All @@ -39,7 +39,7 @@ func init() {
ExamplesCmd.AddCommand(benchmarkCmd)
}

func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error {
func runChunkedWriterBenchmark(_ *cobra.Command, _ []string) error {
payload := make([]byte, payloadSize)
_, err := rand.Read(payload)
if err != nil {
Expand All @@ -53,7 +53,6 @@ func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error {
if err != nil {
return fmt.Errorf("failed to get public key from KAS: %w", err)
}
var attrs []*policy.Value

simpleyKey := &policy.SimpleKasKey{
KasUri: platformEndpoint,
Expand All @@ -65,31 +64,44 @@ func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error {
},
}

attrs = append(attrs, &policy.Value{Fqn: testAttr, KasKeys: []*policy.SimpleKasKey{simpleyKey}, Attribute: &policy.Attribute{Namespace: &policy.Namespace{Name: "example.com"}, Fqn: testAttr}})
writer, err := tdf.NewWriter(context.Background(), tdf.WithDefaultKASForWriter(simpleyKey), tdf.WithInitialAttributes(attrs), tdf.WithSegmentIntegrityAlgorithm(tdf.HS256))
attrs := []*policy.Value{{
Fqn: testAttr,
KasKeys: []*policy.SimpleKasKey{simpleyKey},
Attribute: &policy.Attribute{Namespace: &policy.Namespace{Name: "example.com"}, Fqn: testAttr},
}}

// The package-level constructor rather than SDK.NewChunkedWriter: this
// benchmark talks to one KAS whose key it already fetched, so there is
// nothing for the platform to resolve and no reason to pay for a round trip
// to it inside the timed section.
writer, err := sdk.NewChunkedWriter(context.Background(),
sdk.WithChunkedDefaultKAS(simpleyKey),
sdk.WithChunkedInitialAttributes(attrs),
sdk.WithChunkedSegmentIntegrityAlgorithm(sdk.HS256),
)
if err != nil {
return fmt.Errorf("failed to create writer: %w", err)
}
i := 0
wg := sync.WaitGroup{}

segs := len(payload) / segmentChunk
errs := make([]error, segs)
wg := sync.WaitGroup{}
wg.Add(segs)
start := time.Now()
for i < segs {
segment := i
for segment := range segs {
go func() {
start := i * segmentChunk
end := min(start+segmentChunk, len(payload))
_, err = writer.WriteSegment(context.Background(), segment, payload[start:end])
if err != nil {
fmt.Println(err)
panic(err)
}
wg.Done()
defer wg.Done()
lo := segment * segmentChunk
hi := min(lo+segmentChunk, len(payload))
_, errs[segment] = writer.WriteSegment(context.Background(), segment, payload[lo:hi])
}()
i++
}
wg.Wait()
for i, err := range errs {
if err != nil {
return fmt.Errorf("failed to write segment %d: %w", i, err)
}
}

end := time.Now()
result, err := writer.Finalize(context.Background())
Expand All @@ -98,7 +110,7 @@ func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error {
}
totalTime := end.Sub(start)

fmt.Printf("# Benchmark Experimental TDF Writer Results:\n")
fmt.Printf("# Benchmark Chunked TDF Writer Results:\n")
fmt.Printf("| Metric | Value |\n")
fmt.Printf("|--------------------|--------------|\n")
fmt.Printf("| Payload Size (B) | %d |\n", payloadSize)
Expand Down
67 changes: 33 additions & 34 deletions sdk/chunked_options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,16 @@ import (
"github.com/opentdf/platform/protocol/go/policy"
)

// Each injection-seam option below rejects nil rather than storing it.
// A nil seam is not detectable later: the config field is
// indistinguishable from "not set", so NewChunkedWriter installs no
// default and the nil is dereferenced during writing -- for the
// splitter, not until Finalize, long after the caller has encrypted
// every segment.
// Every option below that takes a pointer or an interface rejects nil
// rather than storing it. A stored nil is not detectable later: the
// config field is indistinguishable from "not set". For an injection
// seam that means NewChunkedWriter installs no default and the nil is
// dereferenced during writing -- for the splitter, not until Finalize,
// long after the caller has encrypted every segment. For the default
// KAS it is worse than a panic, because nothing fails: key access
// silently falls back to the platform base key, and the caller learns
// their data went to a KAS they never named only when a reader cannot
// unwrap it.

// withChunkedArchiveWriterFactory overrides the ZIP archive writer
// factory used by the chunked Writer. The factory must not be nil.
Expand Down Expand Up @@ -55,8 +59,6 @@ func withChunkedClock(clock clock) ChunkedWriterOption {

// WithChunkedInitialAttributes sets attribute values used by Finalize
// when the Finalize call does not supply its own.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedInitialAttributes(values []*policy.Value) ChunkedWriterOption {
return func(c *ChunkedWriterConfig) error {
c.initialAttributes = values
Expand All @@ -65,20 +67,20 @@ func WithChunkedInitialAttributes(values []*policy.Value) ChunkedWriterOption {
}

// WithChunkedDefaultKAS sets the default KAS used by Finalize when
// the Finalize call does not supply its own.
//
// Experimental: not part of the stable SDK API; may change or be removed.
// the Finalize call does not supply its own. The KAS must not be nil:
// omit the option to leave key access to be resolved some other way.
func WithChunkedDefaultKAS(kas *policy.SimpleKasKey) ChunkedWriterOption {
return func(c *ChunkedWriterConfig) error {
if kas == nil {
return errors.New("chunked: default KAS must not be nil")
}
c.initialDefaultKAS = kas
return nil
}
}

// WithChunkedIntegrityAlgorithm sets the algorithm used for the
// manifest root signature.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedIntegrityAlgorithm(algo IntegrityAlgorithm) ChunkedWriterOption {
return func(c *ChunkedWriterConfig) error {
c.integrityAlgorithm = algo
Expand All @@ -90,14 +92,13 @@ func WithChunkedIntegrityAlgorithm(algo IntegrityAlgorithm) ChunkedWriterOption
// chunked Writer. Callers with multi-KAS attribute grants should
// inject a splitter that understands their grant model. The splitter
// must not be nil.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedKeySplitter(splitter KeySplitter) ChunkedWriterOption {
return func(c *ChunkedWriterConfig) error {
if splitter == nil {
return errors.New("chunked: key splitter must not be nil")
}
c.splitter = splitter
c.splitterSet = true
return nil
}
}
Expand All @@ -114,10 +115,20 @@ func withChunkedRand(r io.Reader) ChunkedWriterOption {
}
}

// WithChunkedTDFOptions supplies the key access options — attributes, KAS
// information, preferred wrapping algorithm — that SDK.NewChunkedWriter
// resolves against the platform at Finalize. It has no effect on the
// package-level NewChunkedWriter, which has no platform to resolve against;
// use WithChunkedKeySplitter there.
func WithChunkedTDFOptions(opts ...TDFOption) ChunkedWriterOption {
return func(c *ChunkedWriterConfig) error {
c.tdfOptions = append(c.tdfOptions, opts...)
return nil
}
}

// WithChunkedSegmentIntegrityAlgorithm sets the algorithm used for
// per-segment integrity hashes.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedSegmentIntegrityAlgorithm(algo IntegrityAlgorithm) ChunkedWriterOption {
return func(c *ChunkedWriterConfig) error {
c.segmentIntegrityAlgorithm = algo
Expand All @@ -129,8 +140,6 @@ func WithChunkedSegmentIntegrityAlgorithm(algo IntegrityAlgorithm) ChunkedWriter
// TDF. Each assertion is bound to the payload's aggregate hash, so
// they are signed at Finalize once every segment is in. Assertions
// without their own SigningKey are signed with HS256 over the DEK.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedAssertions(assertions []AssertionConfig) ChunkedFinalizeOption {
return func(c *ChunkedFinalizeConfig) error {
c.assertions = assertions
Expand All @@ -140,8 +149,6 @@ func WithChunkedAssertions(assertions []AssertionConfig) ChunkedFinalizeOption {

// WithChunkedAttributes overrides the writer's initial attributes for
// this Finalize call.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedAttributes(values []*policy.Value) ChunkedFinalizeOption {
return func(c *ChunkedFinalizeConfig) error {
c.attributes = values
Expand All @@ -150,11 +157,13 @@ func WithChunkedAttributes(values []*policy.Value) ChunkedFinalizeOption {
}

// WithChunkedDefaultKASForFinalize overrides the writer's initial
// default KAS for this Finalize call.
//
// Experimental: not part of the stable SDK API; may change or be removed.
// default KAS for this Finalize call. The KAS must not be nil: omit
// the option to keep whatever WithChunkedDefaultKAS set.
func WithChunkedDefaultKASForFinalize(kas *policy.SimpleKasKey) ChunkedFinalizeOption {
return func(c *ChunkedFinalizeConfig) error {
if kas == nil {
return errors.New("chunked: default KAS must not be nil")
}
c.defaultKAS = kas
return nil
}
Expand All @@ -163,8 +172,6 @@ func WithChunkedDefaultKASForFinalize(kas *policy.SimpleKasKey) ChunkedFinalizeO
// WithChunkedEncryptedMetadata attaches AES-GCM-encrypted metadata to
// every KAO in the TDF. The metadata is keyed on the split share and
// only decryptable by a reader that has been granted access.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedEncryptedMetadata(metadata string) ChunkedFinalizeOption {
return func(c *ChunkedFinalizeConfig) error {
c.encryptedMetadata = metadata
Expand All @@ -180,8 +187,6 @@ func WithChunkedEncryptedMetadata(metadata string) ChunkedFinalizeOption {
// WriteSegment, before this option is seen, so on its own this option
// makes Finalize fail with [ErrChunkedVersionHexMismatch]. Pass
// [WithChunkedTargetMode] at construction instead; it sets both.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedExcludeVersion() ChunkedFinalizeOption {
return func(c *ChunkedFinalizeConfig) error {
c.excludeVersion = true
Expand All @@ -200,8 +205,6 @@ func WithChunkedExcludeVersion() ChunkedFinalizeOption {
// cannot be verified by any reader.
//
// An empty mode selects the current format.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedTargetMode(mode string) ChunkedWriterOption {
return func(c *ChunkedWriterConfig) error {
if mode == "" {
Expand All @@ -220,8 +223,6 @@ func WithChunkedTargetMode(mode string) ChunkedWriterOption {
}

// WithChunkedMimeType records the payload MIME type in the manifest.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedMimeType(mimeType string) ChunkedFinalizeOption {
return func(c *ChunkedFinalizeConfig) error {
c.mimeType = mimeType
Expand All @@ -245,8 +246,6 @@ func WithChunkedMimeType(mimeType string) ChunkedFinalizeOption {
// and one that skips a segment with bytes after it would misread every
// segment that follows. For the same reason the caller must
// concatenate each segment's TDFData in ascending index order.
//
// Experimental: not part of the stable SDK API; may change or be removed.
func WithChunkedSegments(indices []int) ChunkedFinalizeOption {
return func(c *ChunkedFinalizeConfig) error {
c.keepSegments = indices
Expand Down
Loading
Loading