From 5fd9a60f70b64f0d1a2adf5a072145478ace2625 Mon Sep 17 00:00:00 2001 From: TianKai Ma Date: Tue, 1 Sep 2026 21:12:07 +0800 Subject: [PATCH] fix(publications): canonicalize content-addressed object MIME --- docs/contracts/publications.json | 2 +- .../server/publication-ingestion-service.ts | 21 +-- .../publication-ingestion-service.test.ts | 149 +++++++++++++++++- tests/unit/publication-object-service.test.ts | 56 +++++++ 4 files changed, 212 insertions(+), 16 deletions(-) diff --git a/docs/contracts/publications.json b/docs/contracts/publications.json index d5d855579..c98e28e23 100644 --- a/docs/contracts/publications.json +++ b/docs/contracts/publications.json @@ -23,7 +23,7 @@ "partial-item-rejection": "A source or item validation error is returned as a per-item rejected result and does not roll back other valid items in the batch. Batch-level identity, manifest, and database failures remain errors.", "objects": "Object keys are generated by the server as publications/{kind}/sha256/{first-two-hex}/{sha256}. Clients cannot choose keys or object metadata. A manifest is at most 32 MiB in this first slice, covering the current crawler's largest public attachments. Metadata ingestion succeeds even when an object is not yet present in R2; plan/complete verifies size, content type, kind, SHA-256 metadata, and the actual bytes (or a provider SHA-256 checksum) before moving the object to verified/linked.", "outbox": "Accepted revisions write a durable PublicationEventOutbox row in the same transaction. A future Queue consumer may derive work from this outbox; the ingestion path does not claim delivery until a real publisher exists.", - "content-type": "Object manifests accept a bare MIME type such as text/html or image/png. Parameters such as charset are rejected so the signed Content-Type and verification contract remain unambiguous.", + "content-type": "Manifests require bare MIME types (for example text/html or image/png); parameters such as charset are rejected. For each (kind, sha256), the first PublicationObject contentType is canonical. Later manifests with the same kind, sha256, and size may use a MIME alias; r2Key and size stay strict. Batch claims, upload plans, presigned PUTs, and completion checks use canonical contentType. Revision replay compares object identity, size, role/order, and alt text, ignoring MIME aliases.", "required-upload-headers": "An upload plan returns the exact required Content-Type, x-amz-meta-kind, and x-amz-meta-sha256 headers. Presigned URLs are short-lived and no R2 credentials are returned." }, "capabilities": { diff --git a/src/features/publications/server/publication-ingestion-service.ts b/src/features/publications/server/publication-ingestion-service.ts index fc0e2efc6..f581654fa 100644 --- a/src/features/publications/server/publication-ingestion-service.ts +++ b/src/features/publications/server/publication-ingestion-service.ts @@ -96,8 +96,10 @@ function normalizeStoredJson(value: unknown) { } type PublicationRevisionSemanticObject = { + // MIME aliases do not change the content-addressed object represented by a + // revision. The stored PublicationObject contentType remains authoritative + // for upload and public-read metadata. altText: string | null; - contentType: string; kind: string; sha256: string; size: number; @@ -167,7 +169,6 @@ function revisionSemanticsFromItem( objects: sortRevisionSemanticObjects( item.objects.map((object) => ({ altText: object.altText ?? null, - contentType: object.contentType, kind: object.kind, sha256: object.sha256, size: object.size, @@ -196,7 +197,6 @@ type StoredPublicationRevision = { objectLinks?: Array<{ altText: string | null; object: { - contentType: string; kind: string; sha256: string; size: number; @@ -226,7 +226,6 @@ function revisionSemanticsFromStored( objects: sortRevisionSemanticObjects( (revision.objectLinks ?? []).map((link) => ({ altText: link.altText, - contentType: link.object?.contentType ?? "", kind: link.role, sha256: link.object?.sha256 ?? "", size: link.object?.size ?? -1, @@ -355,8 +354,7 @@ async function ensureObjectManifest( if ( object.r2Key !== publicationObjectKey(manifest.kind, manifest.sha256) || - object.size !== manifest.size || - object.contentType !== manifest.contentType + object.size !== manifest.size ) { throw new PublicationIngestionBadRequestError( `Object manifest does not match ${manifest.kind}/${manifest.sha256}`, @@ -370,12 +368,12 @@ async function ensureObjectManifest( objectId: object.id, expectedSha256: manifest.sha256, expectedSize: manifest.size, - expectedContentType: manifest.contentType, + expectedContentType: object.contentType, }, update: { expectedSha256: manifest.sha256, expectedSize: manifest.size, - expectedContentType: manifest.contentType, + expectedContentType: object.contentType, }, }); @@ -539,12 +537,7 @@ async function ingestItem( objectLinks: { include: { object: { - select: { - contentType: true, - kind: true, - sha256: true, - size: true, - }, + select: { kind: true, sha256: true, size: true }, }, }, orderBy: [{ sortOrder: "asc" }, { createdAt: "asc" }], diff --git a/tests/unit/publication-ingestion-service.test.ts b/tests/unit/publication-ingestion-service.test.ts index 349e38865..3e9b4c6c0 100644 --- a/tests/unit/publication-ingestion-service.test.ts +++ b/tests/unit/publication-ingestion-service.test.ts @@ -183,7 +183,10 @@ const fake = vi.hoisted(() => { .filter((link) => link.revisionId === revision.id) .map((link) => ({ altText: (link.altText as string | null | undefined) ?? null, - object: state.objects.get(String(link.objectId)) ?? null, + object: + [...state.objects.values()].find( + (object) => object.id === link.objectId, + ) ?? null, role: String(link.role), sortOrder: (link.sortOrder as number | null | undefined) ?? null, })), @@ -436,6 +439,150 @@ describe("publication ingestion transaction", () => { ); }); + it("reuses the canonical MIME and records it on every batch claim", async () => { + const sha256 = + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; + const canonical = { + kind: "asset" as const, + sha256, + size: 3, + contentType: "application/msword", + }; + const alias = { + ...canonical, + contentType: + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + }; + + await ingestPublicationBatch({ + payload: payloadFor( + { + revisionHash: + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + objects: [canonical], + }, + "batch-canonical-mime-original", + ), + principal, + }); + const response = await ingestPublicationBatch({ + payload: payloadFor( + { + revisionHash: + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + observedAt: "2026-09-02", + objects: [alias], + }, + "batch-canonical-mime-alias", + ), + principal, + }); + + expect(response.results[0].status).toBe("updated"); + expect([...fake.state.objects.values()]).toEqual([ + expect.objectContaining({ + kind: canonical.kind, + sha256, + size: canonical.size, + contentType: canonical.contentType, + }), + ]); + expect([...fake.state.claims.values()]).toEqual([ + expect.objectContaining({ expectedContentType: canonical.contentType }), + expect.objectContaining({ expectedContentType: canonical.contentType }), + ]); + }); + + it("rejects an object size mismatch even when the MIME is an alias", async () => { + const sha256 = + "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"; + const original = { + kind: "asset" as const, + sha256, + size: 3, + contentType: "application/msword", + }; + await ingestPublicationBatch({ + payload: payloadFor( + { objects: [original] }, + "batch-size-mismatch-original", + ), + principal, + }); + + await expect( + ingestPublicationBatch({ + payload: payloadFor( + { + observedAt: "2026-09-02", + objects: [ + { + ...original, + size: 4, + contentType: + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + }, + ], + }, + "batch-size-mismatch-alias", + ), + principal, + }), + ).rejects.toBeInstanceOf(PublicationIngestionBadRequestError); + + expect(fake.state.objects.size).toBe(1); + expect(fake.state.claims.size).toBe(1); + expect([...fake.state.objects.values()][0]).toMatchObject({ + size: original.size, + contentType: original.contentType, + }); + }); + + it("accepts a same-revision replay with a MIME alias", async () => { + const sha256 = + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"; + const revisionHash = + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"; + const original = { + kind: "asset" as const, + sha256, + size: 3, + contentType: "application/msword", + }; + await ingestPublicationBatch({ + payload: payloadFor( + { revisionHash, objects: [original] }, + "batch-revision-mime-original", + ), + principal, + }); + + const response = await ingestPublicationBatch({ + payload: payloadFor( + { + revisionHash, + observedAt: "2026-09-02", + objects: [ + { + ...original, + contentType: + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + }, + ], + }, + "batch-revision-mime-alias", + ), + principal, + }); + + expect(response.results[0].status).toBe("updated"); + expect(fake.state.revisions.size).toBe(1); + expect(fake.state.claims.size).toBe(2); + expect([...fake.state.claims.values()][1]).toMatchObject({ + expectedContentType: original.contentType, + }); + }); + it.each([ ["title", { title: "Changed title" }], ["body", { bodyText: "Changed body" }], diff --git a/tests/unit/publication-object-service.test.ts b/tests/unit/publication-object-service.test.ts index 6aa3b54db..0e357ab9d 100644 --- a/tests/unit/publication-object-service.test.ts +++ b/tests/unit/publication-object-service.test.ts @@ -138,6 +138,62 @@ describe("publication object completion", () => { }); }); + it("uses the canonical claim MIME for planning and completion", async () => { + const canonicalContentType = + "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; + const claim = { + expectedContentType: canonicalContentType, + expectedSha256: sha256OfAbc, + expectedSize: 3, + object: { + id: "object-1", + kind: "body_html" as const, + r2Key: `publications/body_html/sha256/ba/${sha256OfAbc}`, + sha256: sha256OfAbc, + status: "pending" as const, + contentType: canonicalContentType, + }, + }; + mocks.batchFindUnique.mockResolvedValue({ objects: [claim] }); + mocks.batchObjectFindFirst.mockResolvedValue(claim); + mocks.bucket.head.mockResolvedValue({ + checksums: { sha256: checksumBytes(sha256OfAbc) }, + customMetadata: { kind: "body_html", sha256: sha256OfAbc }, + httpMetadata: { contentType: canonicalContentType }, + size: 3, + }); + + const planned = await planPublicationObjects({ + principal, + payload: { + batchId: "batch-canonical-content-type", + objects: [{ kind: "body_html", sha256: sha256OfAbc }], + }, + }); + expect(planned.objects[0]?.requiredHeaders).toMatchObject({ + "Content-Type": canonicalContentType, + }); + + await expect( + completePublicationObject({ + principal, + payload: { + batchId: "batch-canonical-content-type", + kind: "body_html", + sha256: sha256OfAbc, + }, + }), + ).resolves.toMatchObject({ status: "linked" }); + expect(mocks.objectUpdate).toHaveBeenCalledWith({ + where: { id: "object-1" }, + data: { + status: "linked", + verifiedAt: expect.any(Date), + lastError: null, + }, + }); + }); + it("plans a large request with one batch lookup and bounded R2 concurrency", async () => { const objectCount = 500; const claims = Array.from({ length: objectCount }, (_, index) => {