Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/contracts/publications.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
"partial-item-rejection": "A source or item validation error is returned as a per-item rejected result and does not roll back other valid items in the batch. Batch-level identity, manifest, and database failures remain errors.",
"objects": "Object keys are generated by the server as publications/{kind}/sha256/{first-two-hex}/{sha256}. Clients cannot choose keys or object metadata. A manifest is at most 32 MiB in this first slice, covering the current crawler's largest public attachments. Metadata ingestion succeeds even when an object is not yet present in R2; plan/complete verifies size, content type, kind, SHA-256 metadata, and the actual bytes (or a provider SHA-256 checksum) before moving the object to verified/linked.",
"outbox": "Accepted revisions write a durable PublicationEventOutbox row in the same transaction. A future Queue consumer may derive work from this outbox; the ingestion path does not claim delivery until a real publisher exists.",
"content-type": "Object manifests accept a bare MIME type such as text/html or image/png. Parameters such as charset are rejected so the signed Content-Type and verification contract remain unambiguous.",
"content-type": "Manifests require bare MIME types (for example text/html or image/png); parameters such as charset are rejected. For each (kind, sha256), the first PublicationObject contentType is canonical. Later manifests with the same kind, sha256, and size may use a MIME alias; r2Key and size stay strict. Batch claims, upload plans, presigned PUTs, and completion checks use canonical contentType. Revision replay compares object identity, size, role/order, and alt text, ignoring MIME aliases.",
"required-upload-headers": "An upload plan returns the exact required Content-Type, x-amz-meta-kind, and x-amz-meta-sha256 headers. Presigned URLs are short-lived and no R2 credentials are returned."
},
"capabilities": {
Expand Down
21 changes: 7 additions & 14 deletions src/features/publications/server/publication-ingestion-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,10 @@ function normalizeStoredJson(value: unknown) {
}

type PublicationRevisionSemanticObject = {
// MIME aliases do not change the content-addressed object represented by a
// revision. The stored PublicationObject contentType remains authoritative
// for upload and public-read metadata.
altText: string | null;
contentType: string;
kind: string;
sha256: string;
size: number;
Expand Down Expand Up @@ -167,7 +169,6 @@ function revisionSemanticsFromItem(
objects: sortRevisionSemanticObjects(
item.objects.map((object) => ({
altText: object.altText ?? null,
contentType: object.contentType,
kind: object.kind,
sha256: object.sha256,
size: object.size,
Expand Down Expand Up @@ -196,7 +197,6 @@ type StoredPublicationRevision = {
objectLinks?: Array<{
altText: string | null;
object: {
contentType: string;
kind: string;
sha256: string;
size: number;
Expand Down Expand Up @@ -226,7 +226,6 @@ function revisionSemanticsFromStored(
objects: sortRevisionSemanticObjects(
(revision.objectLinks ?? []).map((link) => ({
altText: link.altText,
contentType: link.object?.contentType ?? "",
kind: link.role,
sha256: link.object?.sha256 ?? "",
size: link.object?.size ?? -1,
Expand Down Expand Up @@ -355,8 +354,7 @@ async function ensureObjectManifest(

if (
object.r2Key !== publicationObjectKey(manifest.kind, manifest.sha256) ||
object.size !== manifest.size ||
object.contentType !== manifest.contentType
object.size !== manifest.size
) {
throw new PublicationIngestionBadRequestError(
`Object manifest does not match ${manifest.kind}/${manifest.sha256}`,
Expand All @@ -370,12 +368,12 @@ async function ensureObjectManifest(
objectId: object.id,
expectedSha256: manifest.sha256,
expectedSize: manifest.size,
expectedContentType: manifest.contentType,
expectedContentType: object.contentType,
},
update: {
expectedSha256: manifest.sha256,
expectedSize: manifest.size,
expectedContentType: manifest.contentType,
expectedContentType: object.contentType,
},
});

Expand Down Expand Up @@ -539,12 +537,7 @@ async function ingestItem(
objectLinks: {
include: {
object: {
select: {
contentType: true,
kind: true,
sha256: true,
size: true,
},
select: { kind: true, sha256: true, size: true },
},
},
orderBy: [{ sortOrder: "asc" }, { createdAt: "asc" }],
Expand Down
149 changes: 148 additions & 1 deletion tests/unit/publication-ingestion-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,10 @@ const fake = vi.hoisted(() => {
.filter((link) => link.revisionId === revision.id)
.map((link) => ({
altText: (link.altText as string | null | undefined) ?? null,
object: state.objects.get(String(link.objectId)) ?? null,
object:
[...state.objects.values()].find(
(object) => object.id === link.objectId,
) ?? null,
role: String(link.role),
sortOrder: (link.sortOrder as number | null | undefined) ?? null,
})),
Expand Down Expand Up @@ -436,6 +439,150 @@ describe("publication ingestion transaction", () => {
);
});

it("reuses the canonical MIME and records it on every batch claim", async () => {
const sha256 =
"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc";
const canonical = {
kind: "asset" as const,
sha256,
size: 3,
contentType: "application/msword",
};
const alias = {
...canonical,
contentType:
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
};

await ingestPublicationBatch({
payload: payloadFor(
{
revisionHash:
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
objects: [canonical],
},
"batch-canonical-mime-original",
),
principal,
});
const response = await ingestPublicationBatch({
payload: payloadFor(
{
revisionHash:
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
observedAt: "2026-09-02",
objects: [alias],
},
"batch-canonical-mime-alias",
),
principal,
});

expect(response.results[0].status).toBe("updated");
expect([...fake.state.objects.values()]).toEqual([
expect.objectContaining({
kind: canonical.kind,
sha256,
size: canonical.size,
contentType: canonical.contentType,
}),
]);
expect([...fake.state.claims.values()]).toEqual([
expect.objectContaining({ expectedContentType: canonical.contentType }),
expect.objectContaining({ expectedContentType: canonical.contentType }),
]);
});

it("rejects an object size mismatch even when the MIME is an alias", async () => {
const sha256 =
"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd";
const original = {
kind: "asset" as const,
sha256,
size: 3,
contentType: "application/msword",
};
await ingestPublicationBatch({
payload: payloadFor(
{ objects: [original] },
"batch-size-mismatch-original",
),
principal,
});

await expect(
ingestPublicationBatch({
payload: payloadFor(
{
observedAt: "2026-09-02",
objects: [
{
...original,
size: 4,
contentType:
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
},
],
},
"batch-size-mismatch-alias",
),
principal,
}),
).rejects.toBeInstanceOf(PublicationIngestionBadRequestError);

expect(fake.state.objects.size).toBe(1);
expect(fake.state.claims.size).toBe(1);
expect([...fake.state.objects.values()][0]).toMatchObject({
size: original.size,
contentType: original.contentType,
});
});

it("accepts a same-revision replay with a MIME alias", async () => {
const sha256 =
"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee";
const revisionHash =
"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff";
const original = {
kind: "asset" as const,
sha256,
size: 3,
contentType: "application/msword",
};
await ingestPublicationBatch({
payload: payloadFor(
{ revisionHash, objects: [original] },
"batch-revision-mime-original",
),
principal,
});

const response = await ingestPublicationBatch({
payload: payloadFor(
{
revisionHash,
observedAt: "2026-09-02",
objects: [
{
...original,
contentType:
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
},
],
},
"batch-revision-mime-alias",
),
principal,
});

expect(response.results[0].status).toBe("updated");
expect(fake.state.revisions.size).toBe(1);
expect(fake.state.claims.size).toBe(2);
expect([...fake.state.claims.values()][1]).toMatchObject({
expectedContentType: original.contentType,
});
});

it.each([
["title", { title: "Changed title" }],
["body", { bodyText: "Changed body" }],
Expand Down
56 changes: 56 additions & 0 deletions tests/unit/publication-object-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,62 @@ describe("publication object completion", () => {
});
});

it("uses the canonical claim MIME for planning and completion", async () => {
const canonicalContentType =
"application/vnd.openxmlformats-officedocument.wordprocessingml.document";
const claim = {
expectedContentType: canonicalContentType,
expectedSha256: sha256OfAbc,
expectedSize: 3,
object: {
id: "object-1",
kind: "body_html" as const,
r2Key: `publications/body_html/sha256/ba/${sha256OfAbc}`,
sha256: sha256OfAbc,
status: "pending" as const,
contentType: canonicalContentType,
},
};
mocks.batchFindUnique.mockResolvedValue({ objects: [claim] });
mocks.batchObjectFindFirst.mockResolvedValue(claim);
mocks.bucket.head.mockResolvedValue({
checksums: { sha256: checksumBytes(sha256OfAbc) },
customMetadata: { kind: "body_html", sha256: sha256OfAbc },
httpMetadata: { contentType: canonicalContentType },
size: 3,
});

const planned = await planPublicationObjects({
principal,
payload: {
batchId: "batch-canonical-content-type",
objects: [{ kind: "body_html", sha256: sha256OfAbc }],
},
});
expect(planned.objects[0]?.requiredHeaders).toMatchObject({
"Content-Type": canonicalContentType,
});

await expect(
completePublicationObject({
principal,
payload: {
batchId: "batch-canonical-content-type",
kind: "body_html",
sha256: sha256OfAbc,
},
}),
).resolves.toMatchObject({ status: "linked" });
expect(mocks.objectUpdate).toHaveBeenCalledWith({
where: { id: "object-1" },
data: {
status: "linked",
verifiedAt: expect.any(Date),
lastError: null,
},
});
});

it("plans a large request with one batch lookup and bounded R2 concurrency", async () => {
const objectCount = 500;
const claims = Array.from({ length: objectCount }, (_, index) => {
Expand Down
Loading