fix(publications): bound ingestion batches to 100

docs/contracts/publications.json

@@ -19,6 +19,7 @@
     "revision-ordering": "observedAt is the source observation time. An older observation never overwrites the current revision; equal timestamps use the revision hash as a deterministic tie-breaker. A date-only or timezone-less crawler timestamp is interpreted as Asia/Shanghai local time before persistence.",
     "tombstones": "Deletion is explicit: a tombstone item creates a tombstone revision for an existing publication and marks it deleted. Unknown publications cannot be tombstoned. A tombstone result is reported as updated.",
     "batch-idempotency": "Batch identity is scoped by principalKey and batchId. Replaying the same canonical payload returns the stored result without reapplying writes; reusing the pair with another digest returns 409. clientRunId is likewise scoped by principalKey.",
+    "batch-size": "A publication ingestion batch accepts 1-100 items. Larger batches are rejected by the request schema and must be split by the crawler before submission.",
     "partial-item-rejection": "A source or item validation error is returned as a per-item rejected result and does not roll back other valid items in the batch. Batch-level identity, manifest, and database failures remain errors.",
     "objects": "Object keys are generated by the server as publications/{kind}/sha256/{first-two-hex}/{sha256}. Clients cannot choose keys or object metadata. A manifest is at most 32 MiB in this first slice, covering the current crawler's largest public attachments. Metadata ingestion succeeds even when an object is not yet present in R2; plan/complete verifies size, content type, kind, SHA-256 metadata, and the actual bytes (or a provider SHA-256 checksum) before moving the object to verified/linked.",
     "outbox": "Accepted revisions write a durable PublicationEventOutbox row in the same transaction. A future Queue consumer may derive work from this outbox; the ingestion path does not claim delivery until a real publisher exists.",

public/openapi.generated.json

@@ -28943,7 +28943,7 @@
           },
           "items": {
             "minItems": 1,
-            "maxItems": 500,
+            "maxItems": 100,
             "type": "array",
             "items": {
               "anyOf": [

src/features/publications/lib/publication-ingestion-limits.ts

@@ -0,0 +1 @@
+export const PUBLICATION_INGESTION_BATCH_MAX_ITEMS = 100;

src/features/publications/server/publication-ingestion-service.ts

@@ -13,13 +13,15 @@ import { prisma } from "@/lib/db/prisma";
 type TransactionClient = Prisma.TransactionClient;
 
 /**
- * A full publication batch is allowed to contain 500 items. The associated
+ * A full publication batch is allowed to contain 100 items. The associated
  * writes can take about 20 seconds in production, so allow enough time for
  * database contention while staying below the crawler's 60-second request
  * timeout.
  */
 export const PUBLICATION_INGESTION_TRANSACTION_TIMEOUT_MS = 45_000;
 
+export { PUBLICATION_INGESTION_BATCH_MAX_ITEMS } from "@/features/publications/lib/publication-ingestion-limits";
+
 export class PublicationIngestionBadRequestError extends Error {
   readonly code = "publication_ingestion_bad_request";
 }

src/lib/api/schemas/request-publication-ingestion-schemas.ts

@@ -1,5 +1,6 @@
 import * as z from "zod";
 import { parsePublicationDateInput } from "@/features/publications/lib/publication-date";
+import { PUBLICATION_INGESTION_BATCH_MAX_ITEMS } from "@/features/publications/lib/publication-ingestion-limits";
 
 /** Crawler timestamps may be date-only or Shanghai-local naive datetimes. */
 const publicationDateTimeSchema = z
@@ -105,7 +106,10 @@ export const publicationIngestionBatchRequestSchema = z.strictObject({
   batchId: z.string().trim().min(1).max(200),
   observedAt: publicationDateTimeSchema,
   sources: z.array(publicationSourceDescriptorSchema).min(1).max(500),
-  items: z.array(publicationItemSchema).min(1).max(500),
+  items: z
+    .array(publicationItemSchema)
+    .min(1)
+    .max(PUBLICATION_INGESTION_BATCH_MAX_ITEMS),
 });
 
 export const publicationObjectPlanRequestSchema = z.strictObject({

tests/unit/publication-ingestion-contract.test.ts

@@ -1,5 +1,6 @@
 import { describe, expect, it } from "vitest";
 import { parsePublicationDateInput } from "@/features/publications/lib/publication-date";
+import { PUBLICATION_INGESTION_BATCH_MAX_ITEMS } from "@/features/publications/lib/publication-ingestion-limits";
 import { publicationIngestionPayloadDigest } from "@/features/publications/server/publication-ingestion-service";
 import { publicationIngestionBatchRequestSchema } from "@/lib/api/schemas/request-publication-ingestion-schemas";
 import { publicationIngestionBatchResponseSchema } from "@/lib/api/schemas/response-publication-ingestion-schemas";
@@ -28,6 +29,36 @@ describe("publication ingestion contract", () => {
     expect(parsed.success).toBe(true);
   });
 
+  it("accepts 100 items and rejects 101", () => {
+    expect(PUBLICATION_INGESTION_BATCH_MAX_ITEMS).toBe(100);
+    const items = Array.from(
+      { length: PUBLICATION_INGESTION_BATCH_MAX_ITEMS },
+      (_, index) => ({
+        ...fixture.items[0],
+        canonicalUrl: `https://news.ustc.edu.cn/example/${index}.html`,
+        revisionHash: index.toString(16).padStart(64, "0"),
+      }),
+    );
+    expect(
+      publicationIngestionBatchRequestSchema.safeParse({ ...fixture, items })
+        .success,
+    ).toBe(true);
+    const oversizedItems = [
+      ...items,
+      {
+        ...items[0],
+        canonicalUrl: "https://news.ustc.edu.cn/example/101.html",
+        revisionHash: "f".repeat(64),
+      },
+    ];
+    expect(
+      publicationIngestionBatchRequestSchema.safeParse({
+        ...fixture,
+        items: oversizedItems,
+      }).success,
+    ).toBe(false);
+  });
+
   it("keeps bare MIME types and rejects signed Content-Type parameters", () => {
     const base = {
       ...fixture,

tests/unit/publication-ingestion-service.test.ts

@@ -1,4 +1,5 @@
 import { beforeEach, describe, expect, it, vi } from "vitest";
+import { PUBLICATION_INGESTION_BATCH_MAX_ITEMS } from "@/features/publications/lib/publication-ingestion-limits";
 import { publicationIngestionBatchRequestSchema } from "@/lib/api/schemas/request-publication-ingestion-schemas";
 import { PUBLICATION_INGESTION_SERVICE_PRINCIPAL } from "@/lib/auth/service-principal";
 import fixture from "../../docs/contracts/fixtures/publication-batch.json";
@@ -584,17 +585,22 @@ describe("publication ingestion transaction", () => {
       ...parsedFixture,
       batchId: "batch-maximum-size",
       clientRunId: "run-maximum-size",
-      items: Array.from({ length: 500 }, (_, index) => ({
-        ...parsedFixture.items[0],
-        canonicalUrl: `https://news.ustc.edu.cn/example/${index}.html`,
-        revisionHash: index.toString(16).padStart(64, "0"),
-        objects: [],
-      })),
+      items: Array.from(
+        { length: PUBLICATION_INGESTION_BATCH_MAX_ITEMS },
+        (_, index) => ({
+          ...parsedFixture.items[0],
+          canonicalUrl: `https://news.ustc.edu.cn/example/${index}.html`,
+          revisionHash: index.toString(16).padStart(64, "0"),
+          objects: [],
+        }),
+      ),
     });
 
     const response = await ingestPublicationBatch({ payload, principal });
 
-    expect(response.results).toHaveLength(500);
+    expect(response.results).toHaveLength(
+      PUBLICATION_INGESTION_BATCH_MAX_ITEMS,
+    );
     expect(response.results.every(({ status }) => status === "created")).toBe(
       true,
     );