opentdf · dmihalcik-virtru · Sep 1, 2026
@@ -2,14 +2,15 @@ package tdf
 
 import (
 	"errors"
-	"fmt"
+	"io"
 	"os"
 
 	"github.com/opentdf/platform/lib/ocrypto"
 	"github.com/opentdf/platform/otdfctl/cmd/common"
 	"github.com/opentdf/platform/otdfctl/pkg/cli"
+	"github.com/opentdf/platform/otdfctl/pkg/handlers"
 	"github.com/opentdf/platform/otdfctl/pkg/man"
-	"github.com/opentdf/platform/otdfctl/pkg/utils"
+	"github.com/opentdf/platform/otdfctl/pkg/streamio"
 	"github.com/spf13/cobra"
 )
 
@@ -43,55 +44,61 @@ func decryptRun(cmd *cobra.Command, args []string) {
 		sessionKeyAlgorithm = ocrypto.RSA2048Key
 	}
 
-	// check for piped input
-	piped := readPipedStdin()
-
-	// Prefer file argument over piped input over default filename
-	bytesToDecrypt := piped
+	// Prefer the file argument over piped input.
 	var tdfFile string
-	var err error
 	if len(args) > 0 {
 		tdfFile = args[0]
-		bytesToDecrypt, err = utils.ReadBytesFromFile(tdfFile, MaxFileSize)
+	}
+	in, closeIn, err := streamio.OpenSeekable(tdfFile)
+	switch {
+	case errors.Is(err, streamio.ErrNoInput):
+		cli.ExitWithError("Must provide ONE of the following to decrypt: [file argument, stdin input]", err)
+	case err != nil:
+		cli.ExitWithError("Failed to read file:", err)
+	}
+	defer closeIn()
+
+	// Resolve the destination before decrypting, so the plaintext streams
+	// straight to it rather than accumulating in memory first.
+	var dest io.Writer = os.Stdout
+	var outFile *streamio.OutputFile
+	if output != "" {
+		outFile, err = streamio.NewOutputFile(output)
 		if err != nil {
-			cli.ExitWithError("Failed to read file:", err)
+			closeIn()
+			cli.ExitWithError("Failed to write decrypted data to file", err)
 		}
+		defer outFile.Cleanup()
+		dest = outFile
 	}
 
-	if len(bytesToDecrypt) == 0 {
-		cli.ExitWithError("Must provide ONE of the following to decrypt: [file argument, stdin input]", errors.New("no input provided"))
+	// cli.ExitWithError calls os.Exit, which skips deferred functions, so both
+	// the spooled input and the partial output have to be discarded first.
+	fail := func(msg string, err error) {
+		closeIn()
+		if outFile != nil {
+			outFile.Cleanup()
+		}
+		cli.ExitWithError(msg, err)
 	}
 
 	ignoreAllowlist := len(kasAllowList) == 1 && kasAllowList[0] == "*"
 
-	decrypted, err := h.DecryptBytes(
-		c.Context(),
-		bytesToDecrypt,
-		assertionVerification,
-		disableAssertionVerification,
-		sessionKeyAlgorithm,
-		kasAllowList,
-		ignoreAllowlist,
-		nil,
-	)
+	err = h.Decrypt(c.Context(), dest, in, handlers.DecryptOptions{
+		AssertionVerificationKeysFile: assertionVerification,
+		DisableAssertionCheck:         disableAssertionVerification,
+		SessionKeyAlgorithm:           sessionKeyAlgorithm,
+		KASAllowList:                  kasAllowList,
+		IgnoreAllowlist:               ignoreAllowlist,
+	})
 	if err != nil {
-		cli.ExitWithError("Failed to decrypt file", err)
+		fail("Failed to decrypt file", err)
 	}
 
-	if output == "" {
-		//nolint:forbidigo // printing decrypted content to stdout
-		fmt.Print(decrypted.String())
-		return
-	}
-	// Here 'output' is the filename given with -o
-	f, err := os.Create(output)
-	if err != nil {
-		cli.ExitWithError("Failed to write decrypted data to file", err)
-	}
-	defer f.Close()
-	_, err = f.Write(decrypted.Bytes())
-	if err != nil {
-		cli.ExitWithError("Failed to write decrypted data to file", err)
+	if outFile != nil {
+		if err := outFile.Commit(); err != nil {
+			fail("Failed to write decrypted data to file", err)
+		}
 	}
 }
 

@@ -1,39 +1,8 @@
 package tdf
 
-import (
-	"io"
-	"os"
-
-	"github.com/opentdf/platform/otdfctl/pkg/cli"
-	"github.com/opentdf/platform/otdfctl/pkg/streamio"
-)
-
 const (
-	Size1MB     = 1024 * 1024
-	MaxFileSize = int64(10 * 1024 * 1024 * 1024) // 10 GB
-	TDF         = "TDF"
+	Size1MB = 1024 * 1024
+	TDF     = "TDF"
 	// GroupID is the group ID for TDF commands
 	GroupID = TDF
 )
-
-// readPipedStdin returns the whole of piped stdin, or nil when stdin is a
-// terminal or an empty redirect.
-//
-// Detection is delegated to streamio.PipeReader so there is a single answer to
-// "is there piped input?" across the CLI. The read itself is still unbounded;
-// callers that must not hold the payload in memory should use
-// streamio.OpenSeekable instead.
-func readPipedStdin() []byte {
-	r, ok, err := streamio.PipeReader(os.Stdin)
-	if err != nil {
-		cli.ExitWithError("failed to scan bytes from stdin", err)
-	}
-	if !ok {
-		return nil
-	}
-	buf, err := io.ReadAll(r)
-	if err != nil {
-		cli.ExitWithError("failed to scan bytes from stdin", err)
-	}
-	return buf
-}
@@ -54,12 +54,22 @@ runs:
         # suite while other files still create unnamespaced policy fixtures.
         bats --tap e2e --filter-tags namespaced_policy_migration | tee e2e/bats-results.tap
 
+        # Then the streaming suite, also on its own. It round-trips with no
+        # attributes, which falls back to the platform base key, and
+        # key-base.bats sets one pointing at a KAS that does not resolve and
+        # cannot unset it afterwards -- a base key can be replaced but not
+        # cleared. Anything unattributed scheduled after that file produces an
+        # undecryptable TDF, so this has to run first rather than race for a
+        # slot. Running alone also keeps the 1 GiB peak-RSS case from measuring
+        # itself against three neighbours competing for the same memory.
+        bats --tap e2e --filter-tags payload_streaming | tee -a e2e/bats-results.tap
+
         if command -v parallel >/dev/null 2>&1; then
           echo "GNU parallel found, running remaining tests in parallel"
-          bats --tap e2e --filter-tags '!namespaced_policy_migration' --jobs 4 --no-parallelize-within-files --no-tempdir-cleanup | tee -a e2e/bats-results.tap
+          bats --tap e2e --filter-tags '!namespaced_policy_migration,!payload_streaming' --jobs 4 --no-parallelize-within-files --no-tempdir-cleanup | tee -a e2e/bats-results.tap
         else
           echo "GNU parallel not found, running remaining tests sequentially"
-          bats --tap e2e --filter-tags '!namespaced_policy_migration' | tee -a e2e/bats-results.tap
+          bats --tap e2e --filter-tags '!namespaced_policy_migration,!payload_streaming' | tee -a e2e/bats-results.tap
         fi
       env:
         # Define 'bats' install location in ubuntu

@@ -0,0 +1,160 @@
+#!/usr/bin/env bats
+
+# bats file_tags=payload_streaming
+
+# Streaming encrypt/decrypt/inspect (DSPX-4499).
+#
+# These live outside encrypt-decrypt.bats deliberately. That file carries a
+# file-level skip pending the namespaced-subject-mappings migration, so anything
+# added to it would not run. None of the cases here need an entitlement: the
+# round-trips encrypt with no attributes, and the two failure cases are forced
+# with an unresolvable attribute FQN and a KAS allowlist that excludes the
+# platform, neither of which requires policy fixtures.
+#
+# The payload_streaming tag exists so action.yaml can run this file before the
+# parallel batch, and it is load-bearing. Encrypting with no attributes falls
+# back to the platform base key, and key-base.bats sets one pointing at
+# https://test-kas-for-base-keys.com, which does not resolve. It cannot put
+# things back: a base key can be replaced but not cleared, so its teardown
+# leaves the platform unable to decrypt anything unattributed for the rest of
+# the run. encrypt-decrypt.bats would hit the same wall today if it were not
+# skipped. Running before key-base.bats is a workaround, not a fix -- the leak
+# is worth closing on its own.
+
+setup_file() {
+  export HOST=http://localhost:8080
+  export CREDSFILE=creds.json
+  echo -n '{"clientId":"opentdf","clientSecret":"secret"}' >"$CREDSFILE"
+  export WITH_CREDS="--with-client-creds-file $CREDSFILE"
+  export COMMON="--host $HOST --tls-no-verify $WITH_CREDS"
+
+  export SECRET_TEXT="my special streaming secret"
+}
+
+setup() {
+  bats_load_library bats-support
+  bats_load_library bats-assert
+
+  PLAIN="$BATS_TEST_TMPDIR/payload.txt"
+  TDF_OUT="$BATS_TEST_TMPDIR/payload.txt.tdf"
+  RESULT="$BATS_TEST_TMPDIR/payload.out"
+  printf '%s\n' "$SECRET_TEXT" >"$PLAIN"
+}
+
+# Baseline: both ends are seekable files, so nothing is spooled.
+@test "roundtrip TDF3, no attributes, file to file" {
+  ./otdfctl encrypt -o "$TDF_OUT" $COMMON "$PLAIN"
+  ./otdfctl decrypt -o "$RESULT" $COMMON "$TDF_OUT"
+  diff "$PLAIN" "$RESULT"
+}
+
+@test "roundtrip TDF3, no attributes, file to stdout" {
+  ./otdfctl encrypt $COMMON "$PLAIN" >"$TDF_OUT"
+  ./otdfctl decrypt -o "$RESULT" $COMMON "$TDF_OUT"
+  diff "$PLAIN" "$RESULT"
+}
+
+# The fully piped form is the one documented in docs/man/encrypt/_index.md, and
+# the one with no seekable input on either end.
+@test "roundtrip TDF3, stdin to stdout, fully piped" {
+  run bash -c "echo '$SECRET_TEXT' | ./otdfctl encrypt $COMMON | ./otdfctl decrypt $COMMON"
+  assert_success
+  assert_output --partial "$SECRET_TEXT"
+}
+
+# A TDF's manifest lives at the end of the archive, so decrypt spools a pipe to
+# disk to get a seekable view. Verify it round-trips and removes the spool.
+@test "roundtrip TDF3, decrypt reading the TDF from stdin" {
+  ./otdfctl encrypt -o "$TDF_OUT" $COMMON "$PLAIN"
+  # Scope TMPDIR to this test so the leftover check cannot see another test's
+  # spool, and cannot be fooled by one either.
+  TMPDIR="$BATS_TEST_TMPDIR" ./otdfctl decrypt $COMMON <"$TDF_OUT" >"$RESULT"
+  diff "$PLAIN" "$RESULT"
+
+  run bash -c "ls $BATS_TEST_TMPDIR/otdfctl-spool-* 2>/dev/null | wc -l"
+  assert_output "0"
+}
+
+@test "inspect reads a TDF from a file and from stdin" {
+  ./otdfctl encrypt -o "$TDF_OUT" $COMMON "$PLAIN"
+
+  run bash -c "./otdfctl inspect $COMMON '$TDF_OUT' | jq -r '.manifest.protocol'"
+  assert_success
+  assert_output "zip"
+
+  run bash -c "TMPDIR='$BATS_TEST_TMPDIR' ./otdfctl inspect $COMMON < '$TDF_OUT' | jq -r '.manifest.protocol'"
+  assert_success
+  assert_output "zip"
+
+  # inspect spools piped input too, and exits via os.Exit on the success path.
+  run bash -c "ls $BATS_TEST_TMPDIR/otdfctl-spool-* 2>/dev/null | wc -l"
+  assert_output "0"
+}
+
+# An empty redirect is 'no input', not 'a zero-byte payload'. Presence is
+# detected with a peek rather than a read, so this must stay an error.
+@test "encrypt rejects empty stdin" {
+  run bash -c "./otdfctl encrypt $COMMON < /dev/null"
+  assert_failure
+}
+
+@test "decrypt rejects empty stdin" {
+  run bash -c "./otdfctl decrypt $COMMON < /dev/null"
+  assert_failure
+}
+
+# Output goes to a temp sibling and is renamed only on success, so a failed run
+# must leave neither a partial .tdf nor the temp file behind.
+@test "encrypt leaves no output behind when it fails" {
+  run bash -c "echo '$SECRET_TEXT' | ./otdfctl encrypt -o '$TDF_OUT' $COMMON -a 'https://streaming-does-not-exist.io/attr/nope/value/nope'"
+  assert_failure
+  [ ! -f "$TDF_OUT" ]
+
+  run bash -c "ls $BATS_TEST_TMPDIR/.payload.txt.tdf.tmp-* 2>/dev/null | wc -l"
+  assert_output "0"
+}
+
+@test "decrypt leaves no output behind when it fails" {
+  ./otdfctl encrypt -o "$TDF_OUT" $COMMON "$PLAIN"
+
+  # An allowlist with no entry for the platform KAS fails the rewrap.
+  run ./otdfctl decrypt -o "$RESULT" $COMMON --kas-allowlist "https://nowhere.example.com" "$TDF_OUT"
+  assert_failure
+  [ ! -f "$RESULT" ]
+
+  run bash -c "ls $BATS_TEST_TMPDIR/.payload.out.tmp-* 2>/dev/null | wc -l"
+  assert_output "0"
+}
+
+# The point of DSPX-4499: peak RSS is bounded by segment size, not payload size.
+# Needs GNU time for 'Maximum resident set size'; BSD/shell time cannot report it.
+@test "encrypt and decrypt peak memory stay bounded on a large payload" {
+  GNU_TIME=$(command -v gtime || command -v /usr/bin/time)
+  if [ -z "$GNU_TIME" ] || ! $GNU_TIME -v true 2>&1 | grep -q "Maximum resident set size"; then
+    skip "GNU time not available"
+  fi
+
+  local big="$BATS_TEST_TMPDIR/big.bin"
+  local bigtdf="$BATS_TEST_TMPDIR/big.bin.tdf"
+  local bigout="$BATS_TEST_TMPDIR/big.out"
+  local enclog="$BATS_TEST_TMPDIR/enc.log"
+  local declog="$BATS_TEST_TMPDIR/dec.log"
+
+  # 1 GiB. The buffered implementation peaked around 3.6x this for both commands.
+  dd if=/dev/zero of="$big" bs=1048576 count=1024 status=none
+
+  $GNU_TIME -v -o "$enclog" ./otdfctl encrypt -o "$bigtdf" $COMMON "$big"
+  $GNU_TIME -v -o "$declog" ./otdfctl decrypt -o "$bigout" $COMMON "$bigtdf"
+  cmp "$big" "$bigout"
+
+  local enc_kb dec_kb
+  enc_kb=$(grep "Maximum resident set size" "$enclog" | grep -o '[0-9]*')
+  dec_kb=$(grep "Maximum resident set size" "$declog" | grep -o '[0-9]*')
+  rm -f "$big" "$bigtdf" "$bigout"
+
+  echo "peak RSS: encrypt ${enc_kb} KB, decrypt ${dec_kb} KB"
+  # 512 MiB leaves generous headroom over the ~66 MiB a 1 MiB payload used, while
+  # still failing loudly on any return to whole-payload buffering.
+  [ "$enc_kb" -lt 524288 ]
+  [ "$dec_kb" -lt 524288 ]
+}