diff --git a/docs/contracts/publications.json b/docs/contracts/publications.json index 2a5d4a3d0..d5d855579 100644 --- a/docs/contracts/publications.json +++ b/docs/contracts/publications.json @@ -19,6 +19,7 @@ "revision-ordering": "observedAt is the source observation time. An older observation never overwrites the current revision; equal timestamps use the revision hash as a deterministic tie-breaker. A date-only or timezone-less crawler timestamp is interpreted as Asia/Shanghai local time before persistence.", "tombstones": "Deletion is explicit: a tombstone item creates a tombstone revision for an existing publication and marks it deleted. Unknown publications cannot be tombstoned. A tombstone result is reported as updated.", "batch-idempotency": "Batch identity is scoped by principalKey and batchId. Replaying the same canonical payload returns the stored result without reapplying writes; reusing the pair with another digest returns 409. clientRunId is likewise scoped by principalKey.", + "batch-size": "A publication ingestion batch accepts 1-100 items. Larger batches are rejected by the request schema and must be split by the crawler before submission.", "partial-item-rejection": "A source or item validation error is returned as a per-item rejected result and does not roll back other valid items in the batch. Batch-level identity, manifest, and database failures remain errors.", "objects": "Object keys are generated by the server as publications/{kind}/sha256/{first-two-hex}/{sha256}. Clients cannot choose keys or object metadata. A manifest is at most 32 MiB in this first slice, covering the current crawler's largest public attachments. Metadata ingestion succeeds even when an object is not yet present in R2; plan/complete verifies size, content type, kind, SHA-256 metadata, and the actual bytes (or a provider SHA-256 checksum) before moving the object to verified/linked.", "outbox": "Accepted revisions write a durable PublicationEventOutbox row in the same transaction. A future Queue consumer may derive work from this outbox; the ingestion path does not claim delivery until a real publisher exists.", diff --git a/public/openapi.generated.json b/public/openapi.generated.json index f7b47740d..502b2292d 100644 --- a/public/openapi.generated.json +++ b/public/openapi.generated.json @@ -28943,7 +28943,7 @@ }, "items": { "minItems": 1, - "maxItems": 500, + "maxItems": 100, "type": "array", "items": { "anyOf": [ diff --git a/src/features/publications/lib/publication-ingestion-limits.ts b/src/features/publications/lib/publication-ingestion-limits.ts new file mode 100644 index 000000000..ea9dfbdb7 --- /dev/null +++ b/src/features/publications/lib/publication-ingestion-limits.ts @@ -0,0 +1 @@ +export const PUBLICATION_INGESTION_BATCH_MAX_ITEMS = 100; diff --git a/src/features/publications/server/publication-ingestion-service.ts b/src/features/publications/server/publication-ingestion-service.ts index 4ba074f0f..fc0e2efc6 100644 --- a/src/features/publications/server/publication-ingestion-service.ts +++ b/src/features/publications/server/publication-ingestion-service.ts @@ -13,13 +13,15 @@ import { prisma } from "@/lib/db/prisma"; type TransactionClient = Prisma.TransactionClient; /** - * A full publication batch is allowed to contain 500 items. The associated + * A full publication batch is allowed to contain 100 items. The associated * writes can take about 20 seconds in production, so allow enough time for * database contention while staying below the crawler's 60-second request * timeout. */ export const PUBLICATION_INGESTION_TRANSACTION_TIMEOUT_MS = 45_000; +export { PUBLICATION_INGESTION_BATCH_MAX_ITEMS } from "@/features/publications/lib/publication-ingestion-limits"; + export class PublicationIngestionBadRequestError extends Error { readonly code = "publication_ingestion_bad_request"; } diff --git a/src/lib/api/schemas/request-publication-ingestion-schemas.ts b/src/lib/api/schemas/request-publication-ingestion-schemas.ts index 31a2cacef..063e356a7 100644 --- a/src/lib/api/schemas/request-publication-ingestion-schemas.ts +++ b/src/lib/api/schemas/request-publication-ingestion-schemas.ts @@ -1,5 +1,6 @@ import * as z from "zod"; import { parsePublicationDateInput } from "@/features/publications/lib/publication-date"; +import { PUBLICATION_INGESTION_BATCH_MAX_ITEMS } from "@/features/publications/lib/publication-ingestion-limits"; /** Crawler timestamps may be date-only or Shanghai-local naive datetimes. */ const publicationDateTimeSchema = z @@ -105,7 +106,10 @@ export const publicationIngestionBatchRequestSchema = z.strictObject({ batchId: z.string().trim().min(1).max(200), observedAt: publicationDateTimeSchema, sources: z.array(publicationSourceDescriptorSchema).min(1).max(500), - items: z.array(publicationItemSchema).min(1).max(500), + items: z + .array(publicationItemSchema) + .min(1) + .max(PUBLICATION_INGESTION_BATCH_MAX_ITEMS), }); export const publicationObjectPlanRequestSchema = z.strictObject({ diff --git a/tests/unit/publication-ingestion-contract.test.ts b/tests/unit/publication-ingestion-contract.test.ts index eabd331b9..2be217421 100644 --- a/tests/unit/publication-ingestion-contract.test.ts +++ b/tests/unit/publication-ingestion-contract.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { parsePublicationDateInput } from "@/features/publications/lib/publication-date"; +import { PUBLICATION_INGESTION_BATCH_MAX_ITEMS } from "@/features/publications/lib/publication-ingestion-limits"; import { publicationIngestionPayloadDigest } from "@/features/publications/server/publication-ingestion-service"; import { publicationIngestionBatchRequestSchema } from "@/lib/api/schemas/request-publication-ingestion-schemas"; import { publicationIngestionBatchResponseSchema } from "@/lib/api/schemas/response-publication-ingestion-schemas"; @@ -28,6 +29,36 @@ describe("publication ingestion contract", () => { expect(parsed.success).toBe(true); }); + it("accepts 100 items and rejects 101", () => { + expect(PUBLICATION_INGESTION_BATCH_MAX_ITEMS).toBe(100); + const items = Array.from( + { length: PUBLICATION_INGESTION_BATCH_MAX_ITEMS }, + (_, index) => ({ + ...fixture.items[0], + canonicalUrl: `https://news.ustc.edu.cn/example/${index}.html`, + revisionHash: index.toString(16).padStart(64, "0"), + }), + ); + expect( + publicationIngestionBatchRequestSchema.safeParse({ ...fixture, items }) + .success, + ).toBe(true); + const oversizedItems = [ + ...items, + { + ...items[0], + canonicalUrl: "https://news.ustc.edu.cn/example/101.html", + revisionHash: "f".repeat(64), + }, + ]; + expect( + publicationIngestionBatchRequestSchema.safeParse({ + ...fixture, + items: oversizedItems, + }).success, + ).toBe(false); + }); + it("keeps bare MIME types and rejects signed Content-Type parameters", () => { const base = { ...fixture, diff --git a/tests/unit/publication-ingestion-service.test.ts b/tests/unit/publication-ingestion-service.test.ts index a26f81a75..349e38865 100644 --- a/tests/unit/publication-ingestion-service.test.ts +++ b/tests/unit/publication-ingestion-service.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import { PUBLICATION_INGESTION_BATCH_MAX_ITEMS } from "@/features/publications/lib/publication-ingestion-limits"; import { publicationIngestionBatchRequestSchema } from "@/lib/api/schemas/request-publication-ingestion-schemas"; import { PUBLICATION_INGESTION_SERVICE_PRINCIPAL } from "@/lib/auth/service-principal"; import fixture from "../../docs/contracts/fixtures/publication-batch.json"; @@ -584,17 +585,22 @@ describe("publication ingestion transaction", () => { ...parsedFixture, batchId: "batch-maximum-size", clientRunId: "run-maximum-size", - items: Array.from({ length: 500 }, (_, index) => ({ - ...parsedFixture.items[0], - canonicalUrl: `https://news.ustc.edu.cn/example/${index}.html`, - revisionHash: index.toString(16).padStart(64, "0"), - objects: [], - })), + items: Array.from( + { length: PUBLICATION_INGESTION_BATCH_MAX_ITEMS }, + (_, index) => ({ + ...parsedFixture.items[0], + canonicalUrl: `https://news.ustc.edu.cn/example/${index}.html`, + revisionHash: index.toString(16).padStart(64, "0"), + objects: [], + }), + ), }); const response = await ingestPublicationBatch({ payload, principal }); - expect(response.results).toHaveLength(500); + expect(response.results).toHaveLength( + PUBLICATION_INGESTION_BATCH_MAX_ITEMS, + ); expect(response.results.every(({ status }) => status === "created")).toBe( true, );