[WIP] feat(auth): expand ECP custom TLS signer signature buffer for PQC

packages/google-auth/google/auth/transport/_custom_tls_signer.py

@@ -156,9 +156,10 @@ def sign_callback(sig, sig_len, tbs, tbs_len):
         digest = _compute_sha256_digest(tbs, tbs_len)
         digestArray = ctypes.c_char * len(digest)
 
-        # reserve 2000 bytes for the signature, shoud be more then enough.
-        # RSA signature is 256 bytes, EC signature is 70~72.
-        sig_holder_len = 2000
+        # Reserve 65536 bytes (64 KiB) for the signature, which is the maximum signature size
+        # permitted by the TLS 1.3 protocol (RFC 8446). This covers all Post-Quantum Cryptography (PQC)
+        # algorithms including ML-DSA-87 (4627 B), SLH-DSA-128f (17088 B), and SLH-DSA-256f (49856 B).
+        sig_holder_len = 65536
         sig_holder = ctypes.create_string_buffer(sig_holder_len)
 
         signature_len = signer_lib.SignForPython(
@@ -169,8 +170,12 @@ def sign_callback(sig, sig_len, tbs, tbs_len):
             sig_holder_len,  # sigHolderLen
         )
 
-        if signature_len == 0:
-            # signing failed, return 0
+        if signature_len <= 0 or signature_len > sig_holder_len:
+            _LOGGER.error(
+                "Invalid signature length %d returned from signer library (max %d)",
+                signature_len,
+                sig_holder_len,
+            )
             return 0
 
         sig_len[0] = signature_len

packages/google-auth/tests/transport/test__custom_tls_signer.py

@@ -110,6 +110,45 @@ def test_get_sign_callback():
     assert returned_sign_len.value == mock_sig_len
 
 
+def test_get_sign_callback_pqc_signature_size():
+    # ML-DSA-65 is 3,309 B; SLH-DSA-128f is 17,088 B; SLH-DSA-256f is 49,856 B.
+    for mock_sig_len in (3309, 17088, 49856):
+        mock_signer_lib = mock.MagicMock()
+        mock_signer_lib.SignForPython.return_value = mock_sig_len
+
+        sign_callback = _custom_tls_signer.get_sign_callback(
+            mock_signer_lib, FAKE_ENTERPRISE_CERT_FILE_PATH
+        )
+
+        to_be_signed = ctypes.POINTER(ctypes.c_ubyte)()
+        to_be_signed_len = 32
+        returned_sig_array = (ctypes.c_ubyte * mock_sig_len)()
+        mock_sig_array = ctypes.cast(returned_sig_array, ctypes.POINTER(ctypes.c_ubyte))
+        returned_sign_len = ctypes.c_ulong()
+        mock_sig_len_array = ctypes.byref(returned_sign_len)
+
+        assert sign_callback(
+            mock_sig_array, mock_sig_len_array, to_be_signed, to_be_signed_len
+        )
+        assert returned_sign_len.value == mock_sig_len
+
+
+def test_get_sign_callback_oversized_signature():
+    # Verify that signature lengths exceeding 65536 bytes fail gracefully (return 0).
+    mock_signer_lib = mock.MagicMock()
+    mock_signer_lib.SignForPython.return_value = 70000
+
+    sign_callback = _custom_tls_signer.get_sign_callback(
+        mock_signer_lib, FAKE_ENTERPRISE_CERT_FILE_PATH
+    )
+
+    returned_sig_array = (ctypes.c_ubyte * 100)()
+    mock_sig_array = ctypes.cast(returned_sig_array, ctypes.POINTER(ctypes.c_ubyte))
+    returned_sign_len = ctypes.c_ulong()
+
+    assert not sign_callback(mock_sig_array, ctypes.byref(returned_sign_len), None, 0)
+
+
 def test_get_sign_callback_failed_to_sign():
     # mock signer lib's SignForPython function. Set the sig len to be 0 to
     # indicate the signing failed.