fix(cli): stream encrypt instead of buffering the whole payload

otdfctl/cmd/tdf/encrypt.go

@@ -1,8 +1,10 @@
 package tdf
 
 import (
+	"errors"
 	"io"
 	"log/slog"
+	"mime"
 	"os"
 	"path/filepath"
 	"strings"
@@ -11,8 +13,9 @@ import (
 	"github.com/opentdf/platform/lib/ocrypto"
 	"github.com/opentdf/platform/otdfctl/cmd/common"
 	"github.com/opentdf/platform/otdfctl/pkg/cli"
+	"github.com/opentdf/platform/otdfctl/pkg/handlers"
 	"github.com/opentdf/platform/otdfctl/pkg/man"
-	"github.com/opentdf/platform/otdfctl/pkg/utils"
+	"github.com/opentdf/platform/otdfctl/pkg/streamio"
 	"github.com/spf13/cobra"
 )
 
@@ -24,6 +27,39 @@ var (
 	EncryptCmd = &encryptDoc.Command
 )
 
+// detectMimeType sniffs the payload's type from its head and rewinds, so the
+// whole payload still reaches the encoder.
+//
+// Detection needs only the first megabyte, which is what mimetype is limited to
+// anyway, so this reads a bounded prefix rather than the whole payload.
+func detectMimeType(in io.ReadSeeker, fileExt string) (string, error) {
+	mimetype.SetLimit(Size1MB) // limit to 1MB
+
+	head := make([]byte, Size1MB)
+	// A payload shorter than the sniff window is the common case, not an error.
+	n, err := io.ReadFull(in, head)
+	if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
+		return "", err
+	}
+	if _, err := in.Seek(0, io.SeekStart); err != nil {
+		return "", err
+	}
+
+	// defaults to application/octet-stream if nothing is recognized
+	detected := mimetype.Detect(head[:n]).String()
+	if detected == "application/octet-stream" && fileExt != "" {
+		// mime.TypeByExtension is the extension lookup. mimetype.Lookup takes a
+		// MIME type string, so passing it a bare extension always returned nil
+		// and dereferencing that panicked — which is what happened for any file
+		// whose contents were unrecognizable and whose name had an extension.
+		// An extension with no known type leaves octet-stream in place.
+		if byExt := mime.TypeByExtension("." + fileExt); byExt != "" {
+			detected = byExt
+		}
+	}
+	return detected, nil
+}
+
 func encryptRun(cmd *cobra.Command, args []string) {
 	c := cli.New(cmd, args, cli.WithPrintJSON())
 	h := common.NewHandler(c)
@@ -57,13 +93,16 @@ func encryptRun(cmd *cobra.Command, args []string) {
 		wrappingKeyAlgorithm = ocrypto.RSA2048Key
 	}
 
-	piped := readPipedStdin()
+	piped, hasPiped, err := streamio.PipeReader(os.Stdin)
+	if err != nil {
+		cli.ExitWithError("failed to scan bytes from stdin", err)
+	}
 
 	inputCount := 0
 	if filePath != "" {
 		inputCount++
 	}
-	if len(piped) > 0 {
+	if hasPiped {
 		inputCount++
 	}
 
@@ -76,71 +115,84 @@ func encryptRun(cmd *cobra.Command, args []string) {
 		cliExit("ONLY ONE")
 	}
 
-	// prefer filepath argument over stdin input
-	bytesSlice := piped
-	var err error
+	// The SDK seeks to the end of the payload to size it, so the input has to be
+	// seekable. A file already is; a pipe is spooled to disk, which trades the
+	// temporary file for the memory a whole-payload read used to cost.
+	var in io.ReadSeeker
+	var cleanup func()
 	if filePath != "" {
-		bytesSlice, err = utils.ReadBytesFromFile(filePath, MaxFileSize)
+		f, err := os.Open(filePath)
 		if err != nil {
 			cli.ExitWithError("Failed to read file:", err)
 		}
-	}
-
-	// auto-detect mime type if not provided
-	if fileMimeType == "" {
-		slog.Debug("detecting mime type of file")
-		// get the mime type of the file
-		mimetype.SetLimit(Size1MB) // limit to 1MB
-		m := mimetype.Detect(bytesSlice)
-		// default to application/octet-stream if no mime type is detected
-		fileMimeType = m.String()
-
-		if fileMimeType == "application/octet-stream" {
-			if fileExt != "" {
-				fileMimeType = mimetype.Lookup(fileExt).String()
-			}
+		in, cleanup = f, func() { f.Close() }
+	} else {
+		f, spoolCleanup, err := streamio.Spool(piped)
+		if err != nil {
+			cli.ExitWithError("Failed to read stdin:", err)
 		}
+		in, cleanup = f, spoolCleanup
 	}
-	slog.Debug("encrypting file",
-		slog.Int("file_len", len(bytesSlice)),
-		slog.String("mime_type", fileMimeType),
-	)
-
-	// Do the encryption
-	encrypted, err := h.EncryptBytes(
-		tdfType,
-		bytesSlice,
-		attrValues,
-		fileMimeType,
-		kasURLPath,
-		assertions,
-		wrappingKeyAlgorithm,
-		targetMode,
-	)
-	if err != nil {
-		cli.ExitWithError("Failed to encrypt", err)
-	}
-
-	// Find the destination as the output flag filename or stdout
-	var dest *os.File
+	// cli.ExitWithError calls os.Exit, which skips deferred functions, so every
+	// exit below goes through fail() to discard the spool and any partial output.
+	defer cleanup()
+
+	// Resolve the destination before encrypting, so the payload streams straight
+	// to it rather than accumulating in memory first.
+	var dest io.Writer
+	var tdfFile *streamio.OutputFile
 	if out != "" {
 		// make sure output ends in .tdf extension
 		if !strings.HasSuffix(out, ".tdf") {
 			out += ".tdf"
 		}
-		tdfFile, err := os.Create(out)
+		tdfFile, err = streamio.NewOutputFile(out)
 		if err != nil {
+			cleanup()
 			cli.ExitWithError("Failed to write encrypted file "+out, err)
 		}
-		defer tdfFile.Close()
+		defer tdfFile.Cleanup()
 		dest = tdfFile
 	} else {
 		dest = os.Stdout
 	}
 
-	_, e := io.Copy(dest, encrypted)
-	if e != nil {
-		cli.ExitWithError("Failed to write encrypted data to stdout", e)
+	fail := func(msg string, err error) {
+		if tdfFile != nil {
+			tdfFile.Cleanup()
+		}
+		cleanup()
+		cli.ExitWithError(msg, err)
+	}
+
+	// auto-detect mime type if not provided
+	if fileMimeType == "" {
+		slog.Debug("detecting mime type of file")
+		fileMimeType, err = detectMimeType(in, fileExt)
+		if err != nil {
+			fail("Failed to read file:", err)
+		}
+	}
+	slog.Debug("encrypting file", slog.String("mime_type", fileMimeType))
+
+	// Do the encryption
+	err = h.Encrypt(c.Context(), dest, in, handlers.EncryptOptions{
+		TDFType:              tdfType,
+		Attributes:           attrValues,
+		MimeType:             fileMimeType,
+		KASURLPath:           kasURLPath,
+		Assertions:           assertions,
+		WrappingKeyAlgorithm: wrappingKeyAlgorithm,
+		TargetMode:           targetMode,
+	})
+	if err != nil {
+		fail("Failed to encrypt", err)
+	}
+
+	if tdfFile != nil {
+		if err := tdfFile.Commit(); err != nil {
+			fail("Failed to write encrypted file "+out, err)
+		}
 	}
 }
 

otdfctl/cmd/tdf/encrypt_test.go

@@ -0,0 +1,67 @@
+package tdf
+
+import (
+	"bytes"
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDetectMimeTypeRewindsForTheEncoder(t *testing.T) {
+	for _, tc := range []struct {
+		name    string
+		content string
+		want    string
+	}{
+		{name: "text", content: "hello, world\n", want: "text/plain; charset=utf-8"},
+		{name: "json", content: `{"a":1}`, want: "application/json"},
+		// Larger than the sniff window, so a detector that consumed the reader
+		// instead of rewinding would truncate the payload.
+		{name: "larger than sniff window", content: strings.Repeat("a", Size1MB+7), want: "text/plain; charset=utf-8"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			in := strings.NewReader(tc.content)
+
+			got, err := detectMimeType(in, "")
+			require.NoError(t, err)
+			assert.Equal(t, tc.want, got)
+
+			// The whole payload must still reach the encoder.
+			all, err := io.ReadAll(in)
+			require.NoError(t, err)
+			assert.Len(t, all, len(tc.content))
+		})
+	}
+}
+
+func TestDetectMimeTypeFallsBackToExtension(t *testing.T) {
+	// Bytes mimetype cannot classify, so the extension decides. ".pdf" is in
+	// Go's builtin table, so this does not depend on the host's mime.types.
+	unrecognized := bytes.Repeat([]byte{0x01, 0x02, 0x03, 0x04}, 8)
+
+	got, err := detectMimeType(bytes.NewReader(unrecognized), "pdf")
+	require.NoError(t, err)
+	assert.Equal(t, "application/pdf", got)
+}
+
+func TestDetectMimeTypeUnknownExtensionStaysOctetStream(t *testing.T) {
+	unrecognized := bytes.Repeat([]byte{0x01, 0x02, 0x03, 0x04}, 8)
+
+	// The previous implementation called mimetype.Lookup(fileExt).String().
+	// Lookup takes a MIME type string, not an extension, so it returned nil for
+	// every extension and this path panicked.
+	got, err := detectMimeType(bytes.NewReader(unrecognized), "zzzznotathing")
+	require.NoError(t, err)
+	assert.Equal(t, "application/octet-stream", got)
+}
+
+func TestDetectMimeTypeEmptyPayload(t *testing.T) {
+	in := strings.NewReader("")
+
+	got, err := detectMimeType(in, "")
+	require.NoError(t, err)
+	assert.Equal(t, "text/plain", got)
+}

otdfctl/cmd/tdf/inspect.go

@@ -2,11 +2,13 @@ package tdf
 
 import (
 	"errors"
+	"log/slog"
 
 	"github.com/opentdf/platform/otdfctl/cmd/common"
 	"github.com/opentdf/platform/otdfctl/pkg/cli"
 	"github.com/opentdf/platform/otdfctl/pkg/handlers"
 	"github.com/opentdf/platform/otdfctl/pkg/man"
+	"github.com/opentdf/platform/otdfctl/pkg/streamio"
 	"github.com/opentdf/platform/sdk"
 	"github.com/spf13/cobra"
 )
@@ -42,17 +44,36 @@ func inspectRun(cmd *cobra.Command, args []string) {
 	h := common.NewHandler(c)
 	defer h.Close()
 
-	data := cli.ReadFromArgsOrPipe(args, nil)
-	if len(data) == 0 {
-		c.ExitWithError("must provide ONE of the following: [file argument, stdin input]", errors.New("no input provided"))
+	var path string
+	if len(args) > 0 {
+		path = args[0]
 	}
+	in, cleanup, err := streamio.OpenSeekable(path)
+	if err != nil {
+		if errors.Is(err, streamio.ErrNoInput) {
+			c.ExitWithError("must provide ONE of the following: [file argument, stdin input]", err)
+		}
+		c.ExitWithError("failed to read input", err)
+	}
+	// cli.ExitWithError calls os.Exit, which does not run deferred functions, so
+	// cleanup is also invoked explicitly before every exit below — including the
+	// successful one, since piped input is spooled to a temporary file.
+	defer cleanup()
 
-	result, errs := h.InspectTDF(data)
+	result, errs := h.InspectTDF(in)
 	for _, err := range errs {
-		if errors.Is(err, handlers.ErrTDFInspectFailNotValidTDF) {
+		switch {
+		case errors.Is(err, handlers.ErrTDFInspectFailNotValidTDF):
+			cleanup()
 			c.ExitWithError("not a valid TDF", err)
-		} else if errors.Is(err, handlers.ErrTDFInspectFailNotInspectable) {
+		case errors.Is(err, handlers.ErrTDFInspectFailNotInspectable):
+			cleanup()
 			c.ExitWithError("failed to inspect TDF", err)
+		default:
+			// Attribute/metadata reads (e.g. a denied or unreachable KAS rewrap)
+			// are non-fatal by design so the manifest still prints, but must not
+			// vanish silently.
+			slog.Warn("inspect: partial result", slog.Any("error", err))
 		}
 	}
 
@@ -76,8 +97,10 @@ func inspectRun(cmd *cobra.Command, args []string) {
 			Attributes: result.Attributes,
 		}
 
+		cleanup()
 		c.ExitWithJSON(m, cli.ExitCodeSuccess)
 	}
+	cleanup()
 	c.ExitWithError("failed to inspect TDF", nil)
 }
 

otdfctl/cmd/tdf/tdf.go

@@ -5,6 +5,7 @@ import (
 	"os"
 
 	"github.com/opentdf/platform/otdfctl/pkg/cli"
+	"github.com/opentdf/platform/otdfctl/pkg/streamio"
 )
 
 const (
@@ -15,17 +16,24 @@ const (
 	GroupID = TDF
 )
 
+// readPipedStdin returns the whole of piped stdin, or nil when stdin is a
+// terminal or an empty redirect.
+//
+// Detection is delegated to streamio.PipeReader so there is a single answer to
+// "is there piped input?" across the CLI. The read itself is still unbounded;
+// callers that must not hold the payload in memory should use
+// streamio.OpenSeekable instead.
 func readPipedStdin() []byte {
-	stat, err := os.Stdin.Stat()
+	r, ok, err := streamio.PipeReader(os.Stdin)
 	if err != nil {
-		cli.ExitWithError("Failed to read stat from stdin", err)
+		cli.ExitWithError("failed to scan bytes from stdin", err)
 	}
-	if (stat.Mode() & os.ModeCharDevice) == 0 {
-		buf, err := io.ReadAll(os.Stdin)
-		if err != nil {
-			cli.ExitWithError("failed to scan bytes from stdin", err)
-		}
-		return buf
+	if !ok {
+		return nil
 	}
-	return nil
+	buf, err := io.ReadAll(r)
+	if err != nil {
+		cli.ExitWithError("failed to scan bytes from stdin", err)
+	}
+	return buf
 }