diff --git a/datahub-api-model/build.gradle b/datahub-api-model/build.gradle index 7cf3165b..2791d7dd 100644 --- a/datahub-api-model/build.gradle +++ b/datahub-api-model/build.gradle @@ -52,6 +52,11 @@ dependencies { compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' + + // Test-only: the module ships the constraint annotations but no provider, so a test that + // asserts a constraint actually fires needs an implementation to run them. testImplementation, + // so nothing framework-shaped reaches the published artifact. + testImplementation 'org.springframework.boot:spring-boot-starter-validation' } // Library module: no main class, so don't build an executable boot jar. diff --git a/datahub-api-model/src/main/java/ai/intellistream/datahub/api/responses/DataWrapper.java b/datahub-api-model/src/main/java/ai/intellistream/datahub/api/responses/DataWrapper.java index 855e356c..d1e46888 100644 --- a/datahub-api-model/src/main/java/ai/intellistream/datahub/api/responses/DataWrapper.java +++ b/datahub-api-model/src/main/java/ai/intellistream/datahub/api/responses/DataWrapper.java @@ -2,9 +2,11 @@ package ai.intellistream.datahub.api.responses; import ai.intellistream.datahub.models.policy.PolicyWarning; +import ai.intellistream.datahub.models.validation.FieldLimits; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonRootName; import jakarta.validation.Valid; +import jakarta.validation.constraints.Size; import tools.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import io.swagger.v3.oas.annotations.media.Schema; @@ -15,8 +17,12 @@ @Schema(name="DataWrapper", description="DataWrapper with items") public class DataWrapper { + // Bounded on the request side: every create/update/delete endpoint that is not GraphDataWrapper + // binds this envelope, and an unbounded items[] is how a caller turns many small entities into + // bulk storage. Responses are never bean-validated, so paging is unaffected. @JacksonXmlElementWrapper(useWrapping = false) @Valid + @Size(max = FieldLimits.BATCH_ITEMS_MAX) private Collection items = new ArrayList<>(); /** diff --git a/datahub-api-model/src/main/java/ai/intellistream/datahub/api/responses/DatapointString.java b/datahub-api-model/src/main/java/ai/intellistream/datahub/api/responses/DatapointString.java index e1b422fd..9b40935a 100644 --- a/datahub-api-model/src/main/java/ai/intellistream/datahub/api/responses/DatapointString.java +++ b/datahub-api-model/src/main/java/ai/intellistream/datahub/api/responses/DatapointString.java @@ -1,9 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 package ai.intellistream.datahub.api.responses; +import ai.intellistream.datahub.models.validation.FieldLimits; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Size; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -42,6 +44,7 @@ public class DatapointString { private String timestamp; @NotBlank + @Size(max = FieldLimits.DATAPOINT_VALUE_MAX) @Schema(description = """ The data point value as a string, interpreted per the timeseries `valueType`: - `BIGINT` — whole number, max 8 bytes. No fractional part. diff --git a/datahub-api-model/src/main/java/ai/intellistream/datahub/api/responses/DatapointsCollection.java b/datahub-api-model/src/main/java/ai/intellistream/datahub/api/responses/DatapointsCollection.java index 4631e6b0..7711a45a 100644 --- a/datahub-api-model/src/main/java/ai/intellistream/datahub/api/responses/DatapointsCollection.java +++ b/datahub-api-model/src/main/java/ai/intellistream/datahub/api/responses/DatapointsCollection.java @@ -3,7 +3,10 @@ import tools.jackson.databind.annotation.JsonSerialize; import ai.intellistream.datahub.json.ToStringSerializer; +import ai.intellistream.datahub.models.validation.FieldLimits; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; +import jakarta.validation.constraints.Size; import lombok.Data; import lombok.Getter; import lombok.Setter; @@ -24,6 +27,10 @@ public class DatapointsCollection { example = "a4545_well_pump_pressure_a") private String externalId; + // @Valid so the per-datapoint constraints actually cascade: without it neither @NotBlank nor the + // value-length cap on DatapointString was ever evaluated on an insert. + @Valid + @Size(max = FieldLimits.DATAPOINTS_PER_COLLECTION_MAX) private List datapoints; @Schema(description = "If more than 100 000 datapoints are returned, a cursor hash value will be returned", diff --git a/datahub-api-model/src/main/java/ai/intellistream/datahub/models/EventModel.java b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/EventModel.java index 55346f8a..b4dd10b4 100644 --- a/datahub-api-model/src/main/java/ai/intellistream/datahub/models/EventModel.java +++ b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/EventModel.java @@ -6,6 +6,8 @@ import ai.intellistream.datahub.helpers.datetime.DateTimeHandler; import com.fasterxml.jackson.annotation.*; import io.swagger.v3.oas.annotations.media.Schema; +import ai.intellistream.datahub.models.validation.BoundedMetadata; +import ai.intellistream.datahub.models.validation.FieldLimits; import ai.intellistream.datahub.models.validation.ForbiddenValues; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; @@ -53,12 +55,14 @@ public class EventModel extends AbstractResource{ /** * Entity specific metadata. A key-value store. */ + @BoundedMetadata @Schema(description = "Event metadata, additional fields you can bind information with.", example = "{\"definition\": \"SAP ORDER PLACED\"}") private Map metadata = new HashMap<>(); /** * The description of the event. */ + @Size(max = FieldLimits.DESCRIPTION_MAX) @Schema(description = "The description of the event.", example = "This event was caused by....") private String description; @@ -100,6 +104,7 @@ public class EventModel extends AbstractResource{ * externalId, or both; the API resolves the missing side and always returns both. One list * rather than two parallel ones is deliberate — parallel id/externalId lists drift. */ + @Size(max = FieldLimits.RELATED_RESOURCES_MAX) @Schema(description = "Resources that this event has a relation to. Supply id, externalId or both; both are returned.", example = "[{\"id\": 34, \"externalId\": \"sensor_abc\"}]") private List relatedResources = new ArrayList<>(); diff --git a/datahub-api-model/src/main/java/ai/intellistream/datahub/models/GeoLocation.java b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/GeoLocation.java index 6d68a936..4437a95c 100644 --- a/datahub-api-model/src/main/java/ai/intellistream/datahub/models/GeoLocation.java +++ b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/GeoLocation.java @@ -3,8 +3,10 @@ import ai.intellistream.datahub.json.GeoLocationDeserializer; import ai.intellistream.datahub.json.GeoLocationSerializer; +import ai.intellistream.datahub.models.validation.FieldLimits; import com.fasterxml.jackson.annotation.JsonIgnore; import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.Size; import tools.jackson.databind.annotation.JsonDeserialize; import tools.jackson.databind.annotation.JsonSerialize; import tools.jackson.databind.json.JsonMapper; @@ -38,6 +40,7 @@ public class GeoLocation { "Polygon", "MultiPolygon", "GeometryCollection"); /** The raw GeoJSON geometry, stored verbatim. */ + @Size(max = FieldLimits.GEOJSON_MAX_CHARS) private String json; public GeoLocation() { diff --git a/datahub-api-model/src/main/java/ai/intellistream/datahub/models/NodeModel.java b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/NodeModel.java index df63c9a5..cb330185 100644 --- a/datahub-api-model/src/main/java/ai/intellistream/datahub/models/NodeModel.java +++ b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/NodeModel.java @@ -3,6 +3,8 @@ import ai.intellistream.datahub.helpers.text.ExternalIds; import ai.intellistream.datahub.json.ToStringSerializer; +import ai.intellistream.datahub.models.validation.BoundedMetadata; +import ai.intellistream.datahub.models.validation.FieldLimits; import ai.intellistream.datahub.models.validation.ForbiddenValues; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; @@ -60,9 +62,11 @@ public abstract class NodeModel extends AbstractResource { @Schema(description = "The name of the object.", example = "klp pipe ws-a1212-dl") private String name; + @BoundedMetadata @Schema(description = "Entity specific metadata. A key-value store.", example = "{\"work_order\": \"wo-sap-12344\"}") private Map metadata = new HashMap<>(); + @Size(max = FieldLimits.DESCRIPTION_MAX) @Schema(description = "The description of the object.", example = "Water stream pipe") private String description; @@ -101,8 +105,9 @@ public abstract class NodeModel extends AbstractResource { */ @NotNull @Size(min = 1, message = "resource.needs.at.least.one.label") + @Size(max = FieldLimits.LABELS_MAX, message = "resource.too.many.labels") @Schema(description = "A list of the labels associated with this node.", example = "[\"resource\", \"PIPE\"]") - private List labels = new ArrayList<>(); + private List<@Size(max = FieldLimits.LABEL_LENGTH_MAX) String> labels = new ArrayList<>(); /** * The unified node-centric relation encoding: the nodes this one is connected to, each with its @@ -111,6 +116,7 @@ public abstract class NodeModel extends AbstractResource { * {@code relationsFrom}). Populated where the graph is loaded (see {@code ResourceNetwork}); empty * otherwise. */ + @Size(max = FieldLimits.RELATED_RESOURCES_MAX) @Schema(description = "Nodes this node is connected to, with relationship type and direction.", example = "[{\"id\": 34, \"externalId\": \"sensor_abc\", \"relationshipType\": \"PUBLISHES_DATA_TO\", \"direction\": \"OUTBOUND\"}]") private List relatedResources = new ArrayList<>(); diff --git a/datahub-api-model/src/main/java/ai/intellistream/datahub/models/RelForm.java b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/RelForm.java index 532cbf91..ba1a61ad 100644 --- a/datahub-api-model/src/main/java/ai/intellistream/datahub/models/RelForm.java +++ b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/RelForm.java @@ -4,8 +4,11 @@ import ai.intellistream.datahub.json.ToStringSerializer; import ai.intellistream.datahub.helpers.text.TextValidator; +import ai.intellistream.datahub.models.validation.BoundedMetadata; +import ai.intellistream.datahub.models.validation.FieldLimits; import ai.intellistream.datahub.validation.resources.RelationshipTypeNotNull; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.Size; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; @@ -41,11 +44,13 @@ public class RelForm { @JsonSerialize(using = ToStringSerializer.class) private Long relationshipTypeId; + @BoundedMetadata private HashMap metadata = new HashMap<>(); @JsonSerialize(using = ToStringSerializer.class) private Long dataSetId; + @Size(max = FieldLimits.DESCRIPTION_MAX) private String description; public void setName(String name){ diff --git a/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/BoundedMetadata.java b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/BoundedMetadata.java new file mode 100644 index 00000000..f91718ef --- /dev/null +++ b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/BoundedMetadata.java @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: Apache-2.0 +package ai.intellistream.datahub.models.validation; + +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Bounds a metadata map: how many entries it may hold, and how long each key and value may be. + * Defaults come from {@link FieldLimits}. + */ +@Constraint(validatedBy = BoundedMetadataValidator.class) +@Target({ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.ANNOTATION_TYPE}) +@Retention(RetentionPolicy.RUNTIME) +public @interface BoundedMetadata { + + String message() default "metadata.too.large"; + + Class[] groups() default {}; + + Class[] payload() default {}; + + int maxEntries() default FieldLimits.METADATA_MAX_ENTRIES; + + int maxKeyLength() default FieldLimits.METADATA_KEY_MAX; + + int maxValueLength() default FieldLimits.METADATA_VALUE_MAX; +} diff --git a/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/BoundedMetadataValidator.java b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/BoundedMetadataValidator.java new file mode 100644 index 00000000..9ac82730 --- /dev/null +++ b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/BoundedMetadataValidator.java @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 +package ai.intellistream.datahub.models.validation; + +import jakarta.validation.ConstraintValidator; +import jakarta.validation.ConstraintValidatorContext; + +import java.util.Map; + +public class BoundedMetadataValidator implements ConstraintValidator> { + + private int maxEntries; + private int maxKeyLength; + private int maxValueLength; + + @Override + public void initialize(BoundedMetadata constraintAnnotation) { + this.maxEntries = constraintAnnotation.maxEntries(); + this.maxKeyLength = constraintAnnotation.maxKeyLength(); + this.maxValueLength = constraintAnnotation.maxValueLength(); + } + + @Override + public boolean isValid(Map value, ConstraintValidatorContext context) { + if (value == null || value.isEmpty()) { + return true; + } + + if (value.size() > maxEntries) { + return fail(context, "metadata.too.many.entries"); + } + + for (Map.Entry entry : value.entrySet()) { + if (entry.getKey() != null && entry.getKey().length() > maxKeyLength) { + return fail(context, "metadata.key.too.long"); + } + if (entry.getValue() != null && entry.getValue().length() > maxValueLength) { + return fail(context, "metadata.value.too.long"); + } + } + return true; + } + + private boolean fail(ConstraintValidatorContext context, String messageKey) { + context.disableDefaultConstraintViolation(); + context.buildConstraintViolationWithTemplate(messageKey).addConstraintViolation(); + return false; + } +} diff --git a/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/EventFields.java b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/EventFields.java index 04f3f078..1e5429a7 100644 --- a/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/EventFields.java +++ b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/EventFields.java @@ -111,6 +111,9 @@ public boolean validateFields(){ } } + SizeRules.checkLength("Event", "event.description.max.length.error", "Description", + this.description.getSet(), FieldLimits.DESCRIPTION_MAX, errors); + if(this.metadata.getSet() != null){ Map metadata = this.metadata.getSet(); if(metadata.containsKey("")){ @@ -119,6 +122,13 @@ public boolean validateFields(){ } } + SizeRules.checkMetadata("Event", "event", this.metadata, errors); + + SizeRules.checkCount("Event", "event.related.resources.too.many", "Related resources", + this.relatedResources.getSet(), FieldLimits.RELATED_RESOURCES_MAX, errors); + SizeRules.checkCount("Event", "event.related.resources.too.many", "Related resources", + this.relatedResources.getAdd(), FieldLimits.RELATED_RESOURCES_MAX, errors); + // An entry with neither side is unresolvable, so reject it here rather than letting the // service raise it — the caller gets one validation response with every bad field in it. Stream.of(this.relatedResources.getSet(), this.relatedResources.getAdd(), this.relatedResources.getRemove()) diff --git a/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/FieldLimits.java b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/FieldLimits.java new file mode 100644 index 00000000..af251787 --- /dev/null +++ b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/FieldLimits.java @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Apache-2.0 +package ai.intellistream.datahub.models.validation; + +/** + * Size ceilings for the free-form fields of the wire contract, in one place so the bean-validation + * annotations, the hand-written update validators and the OpenAPI schema documentation cannot drift + * apart. + * + *

These bound what a single entity can carry. Without them a caller can use {@code description} + * or {@code metadata} as file storage: both were unbounded at every layer, and events are not even + * subject to the incidental Postgres ceilings that nodes inherit from their indexes. + */ +public final class FieldLimits { + + private FieldLimits() { + } + + /** Long enough for a substantial prose description, short enough to be useless as a file. */ + public static final int DESCRIPTION_MAX = 10_000; + + /** Metadata is a tag store, not a document store. */ + public static final int METADATA_MAX_ENTRIES = 256; + + /** Well under the {@code node_metadata.key varchar(1024)} column. */ + public static final int METADATA_KEY_MAX = 128; + + /** Keeps key+value under the ~2.7 KB {@code (node_id, key, value)} btree index-tuple ceiling. */ + public static final int METADATA_VALUE_MAX = 1_024; + + public static final int RELATED_RESOURCES_MAX = 100; + + public static final int LABELS_MAX = 64; + + public static final int LABEL_LENGTH_MAX = 512; + + /** A detailed polygon fits; a payload does not. */ + public static final int GEOJSON_MAX_CHARS = 65_536; + + /** Numeric strings and status codes fit. */ + public static final int DATAPOINT_VALUE_MAX = 64; + + /** Items in one {@code DataWrapper}. Matches the Java SDK's default ingest batch size. */ + public static final int BATCH_ITEMS_MAX = 10_000; + + /** Datapoints in one collection, for numeric series. */ + public static final int DATAPOINTS_PER_COLLECTION_MAX = 100_000; + + /** + * Datapoints in one collection for TEXT/MIXED series. Tighter than the numeric cap because a + * text batch is the one shape that can approach Pulsar's 5 MB per-message ceiling: this keeps a + * worst-case collection at roughly 640 KB of values. Enforced in the service, once the series' + * value type is known — bean validation runs before the series is resolved. + */ + public static final int TEXT_DATAPOINTS_PER_COLLECTION_MAX = 10_000; +} diff --git a/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/ResourceFields.java b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/ResourceFields.java index 41021be1..27ee0032 100644 --- a/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/ResourceFields.java +++ b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/ResourceFields.java @@ -13,8 +13,11 @@ import ai.intellistream.datahub.validation.FieldValidationError; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.stream.Stream; /** * Resource fields mapping for updating the object @@ -84,6 +87,9 @@ public boolean validateFields(){ } } + SizeRules.checkLength("Resource", "resource.description.max.length.error", "Description", + this.description.getSet(), FieldLimits.DESCRIPTION_MAX, errors); + if(this.metadata.getSet() != null){ Map metadata = this.metadata.getSet(); if(metadata.containsKey("")){ @@ -92,6 +98,19 @@ public boolean validateFields(){ } } + SizeRules.checkMetadata("Resource", "resource", this.metadata, errors); + + SizeRules.checkCount("Resource", "resource.too.many.labels", "Labels", + this.labels.getSet(), FieldLimits.LABELS_MAX, errors); + SizeRules.checkCount("Resource", "resource.too.many.labels", "Labels", + this.labels.getAdd(), FieldLimits.LABELS_MAX, errors); + + Stream.of(this.labels.getSet(), this.labels.getAdd()) + .filter(Objects::nonNull) + .flatMap(Collection::stream) + .forEach(label -> SizeRules.checkLength("Resource", "resource.label.max.length.error", + "Label", label, FieldLimits.LABEL_LENGTH_MAX, errors)); + if(this.geoLocation.getSet() != null && !this.geoLocation.getSet().isValidGeoJson()){ errors.add( new FieldValidationError( diff --git a/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/SizeRules.java b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/SizeRules.java new file mode 100644 index 00000000..7a089511 --- /dev/null +++ b/datahub-api-model/src/main/java/ai/intellistream/datahub/models/validation/SizeRules.java @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: Apache-2.0 +package ai.intellistream.datahub.models.validation; + +import ai.intellistream.datahub.helpers.updates.UpdateMapField; +import ai.intellistream.datahub.validation.FieldValidationError; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * The {@link FieldLimits} ceilings for the hand-written update validators. + * + *

Create validates through bean-validation annotations; update validates by hand, and the two + * paths had drifted — description and metadata were bounded on neither, which left update as a way + * to put into an entity exactly what create had started refusing. + */ +public final class SizeRules { + + private SizeRules() { + } + + /** Record an error if a {@code set} string exceeds {@code max}. */ + public static void checkLength(String objectName, String messageKey, String fieldLabel, + String value, int max, List errors) { + if (value == null || value.length() <= max) { + return; + } + errors.add(new FieldValidationError( + objectName, + new String[]{messageKey}, + new Object[]{value.length()}, + fieldLabel + " max length is " + max + " characters.")); + } + + /** Record an error if a {@code set}/{@code add} collection holds more than {@code max} entries. */ + public static void checkCount(String objectName, String messageKey, String fieldLabel, + Collection value, int max, List errors) { + if (value == null || value.size() <= max) { + return; + } + errors.add(new FieldValidationError( + objectName, + new String[]{messageKey}, + new Object[]{value.size()}, + fieldLabel + " may hold at most " + max + " entries.")); + } + + /** + * Bound a metadata update: entry count, key length and value length, on both {@code set} (which + * replaces the map) and {@code add} (which grows it). {@code add} is checked against the same + * entry cap because the resulting total is not knowable here — this bounds what one request can + * push, which is what the abuse case turns on. + */ + public static void checkMetadata(String objectName, String keyPrefix, UpdateMapField metadata, + List errors) { + checkMetadataMap(objectName, keyPrefix, metadata.getSet(), errors); + checkMetadataMap(objectName, keyPrefix, metadata.getAdd(), errors); + } + + private static void checkMetadataMap(String objectName, String keyPrefix, Map map, + List errors) { + if (map == null || map.isEmpty()) { + return; + } + if (map.size() > FieldLimits.METADATA_MAX_ENTRIES) { + errors.add(new FieldValidationError( + objectName, + new String[]{keyPrefix + ".metadata.too.many.entries"}, + new Object[]{map.size()}, + "Metadata may hold at most " + FieldLimits.METADATA_MAX_ENTRIES + " entries.")); + } + for (Map.Entry entry : map.entrySet()) { + if (entry.getKey() != null && entry.getKey().length() > FieldLimits.METADATA_KEY_MAX) { + errors.add(new FieldValidationError( + objectName, + new String[]{keyPrefix + ".metadata.key.too.long"}, + new Object[]{entry.getKey().length()}, + "Metadata key max length is " + FieldLimits.METADATA_KEY_MAX + " characters.")); + } + if (entry.getValue() != null && entry.getValue().length() > FieldLimits.METADATA_VALUE_MAX) { + errors.add(new FieldValidationError( + objectName, + new String[]{keyPrefix + ".metadata.value.too.long"}, + new Object[]{entry.getValue().length()}, + "Metadata value max length is " + FieldLimits.METADATA_VALUE_MAX + " characters.")); + } + } + } +} diff --git a/datahub-api-model/src/test/java/ai/intellistream/datahub/models/validation/FieldSizeCapsTest.java b/datahub-api-model/src/test/java/ai/intellistream/datahub/models/validation/FieldSizeCapsTest.java new file mode 100644 index 00000000..722ed1b5 --- /dev/null +++ b/datahub-api-model/src/test/java/ai/intellistream/datahub/models/validation/FieldSizeCapsTest.java @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: Apache-2.0 +package ai.intellistream.datahub.models.validation; + +import ai.intellistream.datahub.api.responses.DataWrapper; +import ai.intellistream.datahub.api.responses.DatapointString; +import ai.intellistream.datahub.api.responses.DatapointsCollection; +import ai.intellistream.datahub.models.EventModel; +import ai.intellistream.datahub.models.GeoLocation; +import ai.intellistream.datahub.models.RelForm; +import ai.intellistream.datahub.models.Resource; +import jakarta.validation.ConstraintViolation; +import jakarta.validation.Validation; +import jakarta.validation.Validator; +import jakarta.validation.ValidatorFactory; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The size ceilings that stop an entity being used as file storage. + * + *

Every field asserted here was unbounded at every layer: {@code description} and {@code metadata} + * carried no constraint, and {@code items}/{@code datapoints} no count, so one request could carry as + * much as the transport would pass. The at-max cases are asserted alongside the over-max ones because + * a cap that also rejects legitimate input is its own kind of failure. + */ +class FieldSizeCapsTest { + + private static ValidatorFactory factory; + private static Validator validator; + + @BeforeAll + static void setUp() { + factory = Validation.buildDefaultValidatorFactory(); + validator = factory.getValidator(); + } + + @AfterAll + static void tearDown() { + factory.close(); + } + + private static String repeat(int length) { + return "x".repeat(length); + } + + private static Set paths(Set> violations) { + return violations.stream() + .map(v -> v.getPropertyPath().toString()) + .collect(Collectors.toSet()); + } + + private static EventModel validEvent() { + EventModel event = new EventModel(); + event.setExternalId("valid_event_external_id"); + event.setType("Alarm"); + event.setEventTime(ZonedDateTime.parse("2026-01-01T00:00:00Z")); + return event; + } + + private static Resource validResource() { + Resource resource = new Resource(); + resource.setExternalId("valid_resource_external_id"); + resource.setName("valid resource name"); + resource.setLabels(new ArrayList<>(List.of("ASSET"))); + return resource; + } + + // ---- description ------------------------------------------------------------------------- + + @Test + void event_descriptionAtMax_isAccepted() { + EventModel event = validEvent(); + event.setDescription(repeat(FieldLimits.DESCRIPTION_MAX)); + assertTrue(validator.validate(event).isEmpty()); + } + + @Test + void event_descriptionOverMax_isRejected() { + EventModel event = validEvent(); + event.setDescription(repeat(FieldLimits.DESCRIPTION_MAX + 1)); + assertTrue(paths(validator.validate(event)).contains("description")); + } + + @Test + void resource_descriptionOverMax_isRejected() { + Resource resource = validResource(); + resource.setDescription(repeat(FieldLimits.DESCRIPTION_MAX + 1)); + assertTrue(paths(validator.validate(resource)).contains("description")); + } + + @Test + void relation_descriptionOverMax_isRejected() { + RelForm rel = new RelForm(); + rel.setRelationshipType("CONNECTED_TO"); + rel.setFromExternalId("a_from_external_id"); + rel.setToExternalId("a_to_external_id"); + rel.setDescription(repeat(FieldLimits.DESCRIPTION_MAX + 1)); + assertTrue(paths(validator.validate(rel)).contains("description")); + } + + // ---- metadata ---------------------------------------------------------------------------- + + @Test + void event_metadataAtMaxEntries_isAccepted() { + EventModel event = validEvent(); + event.setMetadata(metadataWithEntries(FieldLimits.METADATA_MAX_ENTRIES)); + assertTrue(validator.validate(event).isEmpty()); + } + + @Test + void event_tooManyMetadataEntries_isRejected() { + EventModel event = validEvent(); + event.setMetadata(metadataWithEntries(FieldLimits.METADATA_MAX_ENTRIES + 1)); + assertTrue(paths(validator.validate(event)).contains("metadata")); + } + + @Test + void event_metadataValueOverMax_isRejected() { + EventModel event = validEvent(); + event.setMetadata(new HashMap<>(Map.of("k", repeat(FieldLimits.METADATA_VALUE_MAX + 1)))); + assertTrue(paths(validator.validate(event)).contains("metadata")); + } + + @Test + void event_metadataKeyOverMax_isRejected() { + EventModel event = validEvent(); + event.setMetadata(new HashMap<>(Map.of(repeat(FieldLimits.METADATA_KEY_MAX + 1), "v"))); + assertTrue(paths(validator.validate(event)).contains("metadata")); + } + + @Test + void resource_metadataValueOverMax_isRejected() { + Resource resource = validResource(); + resource.setMetadata(new HashMap<>(Map.of("k", repeat(FieldLimits.METADATA_VALUE_MAX + 1)))); + assertTrue(paths(validator.validate(resource)).contains("metadata")); + } + + @Test + void event_metadataKeyAndValueAtMax_areAccepted() { + EventModel event = validEvent(); + event.setMetadata(new HashMap<>(Map.of( + repeat(FieldLimits.METADATA_KEY_MAX), repeat(FieldLimits.METADATA_VALUE_MAX)))); + assertTrue(validator.validate(event).isEmpty()); + } + + @Test + void event_nullMetadataValue_isTolerated() { + EventModel event = validEvent(); + Map metadata = new HashMap<>(); + metadata.put("k", null); + event.setMetadata(metadata); + assertTrue(validator.validate(event).isEmpty()); + } + + @Test + void event_emptyMetadata_isAccepted() { + EventModel event = validEvent(); + event.setMetadata(new HashMap<>()); + assertTrue(validator.validate(event).isEmpty()); + } + + private static Map metadataWithEntries(int count) { + Map metadata = new HashMap<>(); + IntStream.range(0, count).forEach(i -> metadata.put("key_" + i, "value")); + return metadata; + } + + // ---- labels and related resources -------------------------------------------------------- + + @Test + void resource_tooManyLabels_isRejected() { + Resource resource = validResource(); + resource.setLabels(IntStream.range(0, FieldLimits.LABELS_MAX + 1) + .mapToObj(i -> "label_" + i) + .collect(Collectors.toCollection(ArrayList::new))); + assertTrue(paths(validator.validate(resource)).contains("labels")); + } + + @Test + void resource_labelOverMaxLength_isRejected() { + Resource resource = validResource(); + resource.setLabels(new ArrayList<>(List.of(repeat(FieldLimits.LABEL_LENGTH_MAX + 1)))); + assertFalse(validator.validate(resource).isEmpty()); + } + + @Test + void event_tooManyRelatedResources_isRejected() { + EventModel event = validEvent(); + event.setRelatedResources(IntStream.range(0, FieldLimits.RELATED_RESOURCES_MAX + 1) + .mapToObj(i -> new ai.intellistream.datahub.models.IdCollection()) + .collect(Collectors.toCollection(ArrayList::new))); + assertTrue(paths(validator.validate(event)).contains("relatedResources")); + } + + // ---- geolocation ------------------------------------------------------------------------- + + @Test + void geoLocation_overMaxLength_isRejected() { + // Structurally valid GeoJSON, just far too much of it. + String coordinates = IntStream.range(0, 20_000) + .mapToObj(i -> "[1.0,2.0]") + .collect(Collectors.joining(",")); + GeoLocation geo = new GeoLocation("{\"type\":\"MultiPoint\",\"coordinates\":[" + coordinates + "]}"); + assertTrue(geo.isValidGeoJson(), "precondition: the payload is valid GeoJSON"); + assertTrue(paths(validator.validate(geo)).contains("json")); + } + + // ---- datapoints -------------------------------------------------------------------------- + + @Test + void datapointValueOverMax_isRejected() { + DatapointString datapoint = new DatapointString("1735689600000", repeat(FieldLimits.DATAPOINT_VALUE_MAX + 1)); + assertTrue(paths(validator.validate(datapoint)).contains("value")); + } + + @Test + void datapointConstraintsCascadeThroughTheCollection() { + // Regression: DatapointsCollection.datapoints carried no @Valid, so neither @NotBlank nor the + // value cap on DatapointString was ever evaluated on an insert. + DatapointsCollection collection = new DatapointsCollection(); + collection.setExternalId("a_timeseries"); + collection.setDatapoints(List.of(new DatapointString("1735689600000", repeat(FieldLimits.DATAPOINT_VALUE_MAX + 1)))); + + DataWrapper wrapper = new DataWrapper<>(); + wrapper.setItems(List.of(collection)); + + assertFalse(validator.validate(wrapper).isEmpty()); + } + + @Test + void tooManyDatapointsInOneCollection_isRejected() { + DatapointsCollection collection = new DatapointsCollection(); + collection.setExternalId("a_timeseries"); + collection.setDatapoints(IntStream.range(0, FieldLimits.DATAPOINTS_PER_COLLECTION_MAX + 1) + .mapToObj(i -> new DatapointString("1735689600000", "1.0")) + .toList()); + assertTrue(paths(validator.validate(collection)).contains("datapoints")); + } + + // ---- batch size -------------------------------------------------------------------------- + + @Test + void batchAtMaxItems_isAccepted() { + DataWrapper wrapper = new DataWrapper<>(); + wrapper.setItems(IntStream.range(0, FieldLimits.BATCH_ITEMS_MAX) + .mapToObj(i -> { + EventModel event = validEvent(); + event.setExternalId("event_external_id_" + i); + return event; + }) + .toList()); + assertTrue(validator.validate(wrapper).isEmpty()); + } + + @Test + void batchOverMaxItems_isRejected() { + DataWrapper wrapper = new DataWrapper<>(); + wrapper.setItems(IntStream.range(0, FieldLimits.BATCH_ITEMS_MAX + 1) + .mapToObj(i -> validEvent()) + .toList()); + assertEquals(Set.of("items"), paths(validator.validate(wrapper))); + } +} diff --git a/datahub-api-model/src/test/java/ai/intellistream/datahub/models/validation/UpdateFieldSizeCapsTest.java b/datahub-api-model/src/test/java/ai/intellistream/datahub/models/validation/UpdateFieldSizeCapsTest.java new file mode 100644 index 00000000..b6af9107 --- /dev/null +++ b/datahub-api-model/src/test/java/ai/intellistream/datahub/models/validation/UpdateFieldSizeCapsTest.java @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: Apache-2.0 +package ai.intellistream.datahub.models.validation; + +import ai.intellistream.datahub.validation.FieldValidationError; +import org.junit.jupiter.api.Test; +import tools.jackson.databind.json.JsonMapper; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The update path enforces the same ceilings as create. + * + *

Create validates through annotations, update through these hand-written validators, and the two + * had drifted: neither description nor metadata was bounded here, so update was a way to put into an + * entity exactly what create had started refusing. + * + *

Driven through JSON like {@link RequiredFieldSetNullTest}, because which keys the caller did and + * did not send is what these validators branch on. + */ +class UpdateFieldSizeCapsTest { + + private final JsonMapper mapper = JsonMapper.builder().build(); + + private static boolean mentions(List errors, String messageKey) { + return errors.stream().anyMatch(it -> List.of(it.getCodes()).contains(messageKey)); + } + + private static String repeat(int length) { + return "x".repeat(length); + } + + /** {@code {"k0":"v", "k1":"v", …}} with {@code count} entries. */ + private static String metadataJson(int count) { + return IntStream.range(0, count) + .mapToObj(i -> "\"key_%d\":\"value\"".formatted(i)) + .collect(Collectors.joining(",", "{", "}")); + } + + // ---- events ------------------------------------------------------------------------------ + + @Test + void event_descriptionOverMax_isRejected() { + EventFields fields = mapper.readValue( + "{\"description\": {\"set\": \"%s\"}}".formatted(repeat(FieldLimits.DESCRIPTION_MAX + 1)), + EventFields.class); + assertFalse(fields.validateFields()); + assertTrue(mentions(fields.getErrors(), "event.description.max.length.error")); + } + + @Test + void event_descriptionAtMax_isAccepted() { + EventFields fields = mapper.readValue( + "{\"description\": {\"set\": \"%s\"}}".formatted(repeat(FieldLimits.DESCRIPTION_MAX)), + EventFields.class); + assertTrue(fields.validateFields()); + } + + @Test + void event_tooManyMetadataEntriesOnSet_isRejected() { + EventFields fields = mapper.readValue( + "{\"metadata\": {\"set\": %s}}".formatted(metadataJson(FieldLimits.METADATA_MAX_ENTRIES + 1)), + EventFields.class); + assertFalse(fields.validateFields()); + assertTrue(mentions(fields.getErrors(), "event.metadata.too.many.entries")); + } + + @Test + void event_tooManyMetadataEntriesOnAdd_isRejected() { + // add grows the map, so it has to be bounded too — otherwise the cap is one request away. + EventFields fields = mapper.readValue( + "{\"metadata\": {\"add\": %s}}".formatted(metadataJson(FieldLimits.METADATA_MAX_ENTRIES + 1)), + EventFields.class); + assertFalse(fields.validateFields()); + assertTrue(mentions(fields.getErrors(), "event.metadata.too.many.entries")); + } + + @Test + void event_metadataValueOverMax_isRejected() { + EventFields fields = mapper.readValue( + "{\"metadata\": {\"set\": {\"k\": \"%s\"}}}".formatted(repeat(FieldLimits.METADATA_VALUE_MAX + 1)), + EventFields.class); + assertFalse(fields.validateFields()); + assertTrue(mentions(fields.getErrors(), "event.metadata.value.too.long")); + } + + @Test + void event_metadataKeyOverMax_isRejected() { + EventFields fields = mapper.readValue( + "{\"metadata\": {\"set\": {\"%s\": \"v\"}}}".formatted(repeat(FieldLimits.METADATA_KEY_MAX + 1)), + EventFields.class); + assertFalse(fields.validateFields()); + assertTrue(mentions(fields.getErrors(), "event.metadata.key.too.long")); + } + + // ---- resources --------------------------------------------------------------------------- + + @Test + void resource_descriptionOverMax_isRejected() { + ResourceFields fields = mapper.readValue( + "{\"description\": {\"set\": \"%s\"}}".formatted(repeat(FieldLimits.DESCRIPTION_MAX + 1)), + ResourceFields.class); + assertFalse(fields.validateFields()); + assertTrue(mentions(fields.getErrors(), "resource.description.max.length.error")); + } + + @Test + void resource_metadataValueOverMax_isRejected() { + ResourceFields fields = mapper.readValue( + "{\"metadata\": {\"set\": {\"k\": \"%s\"}}}".formatted(repeat(FieldLimits.METADATA_VALUE_MAX + 1)), + ResourceFields.class); + assertFalse(fields.validateFields()); + assertTrue(mentions(fields.getErrors(), "resource.metadata.value.too.long")); + } + + @Test + void resource_tooManyLabels_isRejected() { + String labels = IntStream.range(0, FieldLimits.LABELS_MAX + 1) + .mapToObj(i -> "\"label_%d\"".formatted(i)) + .collect(Collectors.joining(",", "[", "]")); + ResourceFields fields = mapper.readValue( + "{\"labels\": {\"set\": %s}}".formatted(labels), ResourceFields.class); + assertFalse(fields.validateFields()); + assertTrue(mentions(fields.getErrors(), "resource.too.many.labels")); + } + + @Test + void resource_labelOverMaxLength_isRejected() { + ResourceFields fields = mapper.readValue( + "{\"labels\": {\"add\": [\"%s\"]}}".formatted(repeat(FieldLimits.LABEL_LENGTH_MAX + 1)), + ResourceFields.class); + assertFalse(fields.validateFields()); + assertTrue(mentions(fields.getErrors(), "resource.label.max.length.error")); + } + + @Test + void resource_ordinaryUpdate_isStillAccepted() { + ResourceFields fields = mapper.readValue(""" + {"description": {"set": "a normal description"}, + "metadata": {"add": {"work_order": "wo-sap-12344"}}, + "labels": {"add": ["PIPE"]}}""", ResourceFields.class); + assertTrue(fields.validateFields()); + } +} diff --git a/datahub-api/DATAPOINT_METADATA_CACHE.md b/datahub-api/DATAPOINT_METADATA_CACHE.md new file mode 100644 index 00000000..4cd1ccf8 --- /dev/null +++ b/datahub-api/DATAPOINT_METADATA_CACHE.md @@ -0,0 +1,103 @@ +# TODO: Valkey-cached timeseries metadata for datapoint ingestion + +**Status:** proposed (not yet implemented) +**Module:** `datahub-api` +**Owner:** _unassigned_ + +## Why + +Datapoint ingestion is a hot path, but `TimeseriesService.insertDatapoints()` +(`datahub-api/.../api/services/TimeseriesService.java`, ~line 628) and +`deleteDatapoints()` (~line 936) both resolve the target timeseries from +PostgreSQL on every request via `timeseriesRepository.findByIdOrExternalId(...)`. + +Two costs follow from that: + +1. The methods are `@Transactional`, and because the API's `DataSource` is a + `StatelessRoutingDataSource` handing out **unpooled** `SimpleDriverDataSource` + connections (one fresh physical connect+auth per acquisition, eagerly acquired + at transaction begin), every ingestion request opens a per-tenant Postgres + connection purely to read small, slow-changing metadata. +2. `insertDatapoints()` holds that connection open across a loop of **synchronous, + blocking** `allDatapointProducer.send(...)` calls — DB connection tied up across + Pulsar I/O. + +There is **no dual-write concern** here (the methods perform no Postgres writes, +so there is nothing to roll back, and the inline send is correct — this is +deliberately *not* routed through `AfterCommitMessagePublisher`). The only thing +keeping Postgres on the hot path is the metadata read. + +## What the hot path actually needs from the entity + +From `insertDatapoints` + `addData(...)` + the ACL check, the only fields read are: + +| Field | Used for | +|-------|----------| +| `id` (long) | `DataCollectionString.setId`, producer topic resolution | +| `externalId` (String) | `addData`, `addToLatestValuesCache` | +| `valueType` id (`BIGINT`/`DECIMAL`/`NUMERIC`/`TEXT`) | per-value parse/validation | +| `valueType` name | `DataCollectionString.setValueType` | +| `dataSet` id (nullable) | `dataSecurity.assertCanWrite(ts)` | + +That is the entire cacheable payload — small and stable. + +## Plan + +1. **`TimeseriesMeta` record** — immutable `(id, externalId, valueTypeId, + valueTypeName, dataSetId)`. Place in `datahub-lib-nodep` (or `datahub-library`). + +2. **Extend `ValkeyService`** (`datahub-library/.../services/ValkeyService.java`), + mirroring the existing `fetchLatestDatapoint`/`setLatestDatapoint`/`delete(key)` + pattern (it already has `RedisClient` + `jsonMapper` + `DEFAULT_EXPIRE_TIME = 300`): + - `Optional fetchTimeseriesMeta(String tenantId, String lookupKey)` + - `setTimeseriesMeta(String tenantId, TimeseriesMeta meta)` — writes **both** + lookup keys: `tsmeta:{tenant}:id:{id}` and `tsmeta:{tenant}:eid:{externalId}`, + each with a TTL. + - `evictTimeseriesMeta(String tenantId, long id, String externalId)` — deletes + both keys. + +3. **Rework resolution** in `insertDatapoints` / `deleteDatapoints`: + cache lookup → on miss, a single `findByIdOrExternalId` load → populate cache. + Replace `dataSecurity.assertCanWrite(ts)` with the existing + `dataSecurity.assertCanWriteDataSet(meta.dataSetId())`. Once resolution is + cache-backed, **drop `@Transactional`** — a cache hit touches no Postgres, and a + miss does a single short read. + +4. **Invalidation — must be airtight** (stale dataset/externalId gates a *write* + permission check, so this is security-sensitive). Evict in every metadata-mutating + path: + - `updateTimeseries(...)` (~line 1012) — externalId and dataset can change → + evict **old and new** externalId. + - `deleteTimeseries(...)` (~line 166) — evict, or a deleted series stays + "writable" in cache until TTL. + - `save(...)` (~line 547) — evict-by-externalId on create to clear any stale + entry from a prior delete+recreate of the same externalId. + - Eviction propagates across API instances because Valkey is shared — **but only + if every writer calls it.** Audit for any other path that mutates a timeseries' + dataset or externalId. + +## Gotchas to bake in + +- **Tenant scoping is mandatory.** Keys must include `TenantContext.getTenantId()` + (ids and externalIds are per-tenant). While here, double-check the *existing* + latest-datapoint cache, which keys on a bare `externalId` — possible pre-existing + cross-tenant collision. +- **Dual-key lookup.** Requests resolve by id *or* externalId; write both keys so + either hits, and evict both. +- **TTL bounds a write-ACL staleness window.** With explicit eviction on every write + path, TTL is only a backstop for a missed evict — keep it modest (~300s) since it + gates a permission check. +- **Confirm `valueType` immutability** post-create. If it cannot change, that field + is safe to cache for the full TTL without extra invalidation. +- **Skip negative caching** of "not found" (the `ts == null` → 404 branch) to avoid a + tombstone/eviction dance, unless profiling shows it's needed. +- **Stampede:** an uncached hot series under load yields concurrent misses each doing + a DB load — acceptable; add single-flight only if measured. + +## Out of scope / not changing + +- `ResourceService` create/update/delete and `TimeseriesService` save/update/delete + of *metadata* keep their `@Transactional` + `AfterCommitMessagePublisher` pattern — + those do real Postgres writes (and `delete` paths hold a pessimistic + `lockByIdIn`), so the after-commit publish is load-bearing there. +- This is purely the datapoint **value** ingestion path. diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/ApiDatahubApplication.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/ApiDatahubApplication.java index 65ae0c8a..085bc550 100644 --- a/datahub-api/src/main/java/ai/intellistream/datahub/api/ApiDatahubApplication.java +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/ApiDatahubApplication.java @@ -19,6 +19,7 @@ import io.swagger.v3.oas.models.Paths; import org.springdoc.core.customizers.OpenApiCustomizer; import org.springframework.boot.SpringApplication; +import tools.jackson.core.StreamReadConstraints; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; @@ -119,7 +120,26 @@ backoff; if it persists, contact support. Every error body is safe to log and show to end users; it never contains - credentials or internal stack traces.""" + credentials or internal stack traces. + + ## Limits + + Requests are bounded so no single caller can crowd out the rest. Going + over any of these is a 4xx: the request never becomes acceptable by + retrying it unchanged. + + - **Request body** — 4 MiB, or 16 MiB for `POST /timeseries/data`. + Over that is a **413**; split the batch. + - **Batch size** — 10 000 `items` per request (1000 nodes plus 1000 + relations for `/resources/create`). + - **Data points** — 100 000 per collection, or 10 000 for a `TEXT`/ + `MIXED` series. A single value is at most 64 characters. + - **Free-text fields** — `description` 10 000 characters; `metadata` 256 + entries, keys 128 and values 1024 characters; 64 labels of at most 512 + characters; 100 related resources. + + These bound one request. They are not a licence to send unlimited + requests: sustained volume is what the 429 above is for.""" ), servers = {@Server(url = "https://api-{project}.intellistream.ai", description = "The url is your api-{your-project-name}.intellistream.ai")}, @@ -137,6 +157,17 @@ public class ApiDatahubApplication { public static void main(String[] args) { + // Before the context starts, so every JSON factory built during autoconfiguration inherits + // it. A second line behind RequestBodySizeLimitFilter: the filter bounds a whole body, this + // bounds one absurd scalar or a pathological nesting depth within a legal one. Unlike + // unknown-field strictness (see StrictRequestBodyConfig), a generous size ceiling is safe to + // apply globally — it cannot reject any tenant registry a smaller document would parse. + StreamReadConstraints.overrideDefaultStreamReadConstraints( + StreamReadConstraints.builder() + .maxStringLength(2_000_000) + .maxDocumentLength(64L * 1024 * 1024) + .build()); + SpringApplication app = new SpringApplication(ApiDatahubApplication.class); // Registered here, not in spring.factories, so @SpringBootTest contexts never reach Vault. app.addListeners(new VaultConfigurationLoader( diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/config/LimitsProperties.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/config/LimitsProperties.java new file mode 100644 index 00000000..00388e09 --- /dev/null +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/config/LimitsProperties.java @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.config; + +import ai.intellistream.datahub.models.validation.FieldLimits; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * Binds {@code datahub.limits.*}: the ceilings that keep the api usable when it is reachable by the + * public. The per-field and per-batch caps live in the wire contract ({@link FieldLimits}); what is + * configurable here is deployment policy rather than contract. + */ +@Component +@ConfigurationProperties(prefix = "datahub.limits") +public class LimitsProperties { + + /** + * Largest accepted request body, in bytes. Kept below Pulsar's 5 MB per-message default: an + * event create batch is published as a single message, so the body cap is what keeps that + * message legal. + */ + private long maxBodyBytes = 4L * 1024 * 1024; + + /** + * Largest accepted body for {@code POST /timeseries/data}. Higher than the general cap because a + * full {@link FieldLimits#DATAPOINTS_PER_COLLECTION_MAX} batch of numeric points is around 5 MB + * of JSON on its own. + */ + private long maxBodyBytesDatapoints = 16L * 1024 * 1024; + + public long getMaxBodyBytes() { + return maxBodyBytes; + } + + public void setMaxBodyBytes(long maxBodyBytes) { + this.maxBodyBytes = maxBodyBytes; + } + + public long getMaxBodyBytesDatapoints() { + return maxBodyBytesDatapoints; + } + + public void setMaxBodyBytesDatapoints(long maxBodyBytesDatapoints) { + this.maxBodyBytesDatapoints = maxBodyBytesDatapoints; + } + + private final Rate rate = new Rate(); + private final Quota quota = new Quota(); + private final Lifetime lifetime = new Lifetime(); + private final WebSocket websocket = new WebSocket(); + + public WebSocket getWebsocket() { + return websocket; + } + + /** + * Concurrent WebSocket connections, and subscriptions multiplexed over one of them. Sockets are + * capped separately from requests because a socket's cost is what happens after the handshake: + * a durable subscription holds broker resources whether or not anyone is reading it. + */ + public static class WebSocket { + + private boolean enabled = true; + + private int maxSocketsPerTenant = 10; + private int maxSocketsPerUser = 10; + private int maxSubscriptionsPerSocket = 10; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public int getMaxSocketsPerTenant() { + return maxSocketsPerTenant; + } + + public void setMaxSocketsPerTenant(int maxSocketsPerTenant) { + this.maxSocketsPerTenant = maxSocketsPerTenant; + } + + public int getMaxSocketsPerUser() { + return maxSocketsPerUser; + } + + public void setMaxSocketsPerUser(int maxSocketsPerUser) { + this.maxSocketsPerUser = maxSocketsPerUser; + } + + public int getMaxSubscriptionsPerSocket() { + return maxSubscriptionsPerSocket; + } + + public void setMaxSubscriptionsPerSocket(int maxSubscriptionsPerSocket) { + this.maxSubscriptionsPerSocket = maxSubscriptionsPerSocket; + } + } + + public Rate getRate() { + return rate; + } + + public Quota getQuota() { + return quota; + } + + public Lifetime getLifetime() { + return lifetime; + } + + /** + * Daily ingest allowance per tenant, reset at 00:00 UTC. Overridable per tenant; 0 or negative + * disables one. + */ + public static class Quota { + + private boolean enabled = true; + + private long eventsPerDay = 100_000; + /** Resources, timeseries, datasets, labels, policies and functions share the node table. */ + private long nodesPerDay = 50_000; + private long edgesPerDay = 100_000; + private long datapointsPerDay = 10_000_000; + /** + * Bytes of write-request body. The only quota that really bounds storage growth: an entity + * count does not, because one legitimate entity may be a few hundred KB. + */ + private long ingestBytesPerDay = 1024L * 1024 * 1024; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public long getEventsPerDay() { + return eventsPerDay; + } + + public void setEventsPerDay(long eventsPerDay) { + this.eventsPerDay = eventsPerDay; + } + + public long getNodesPerDay() { + return nodesPerDay; + } + + public void setNodesPerDay(long nodesPerDay) { + this.nodesPerDay = nodesPerDay; + } + + public long getEdgesPerDay() { + return edgesPerDay; + } + + public void setEdgesPerDay(long edgesPerDay) { + this.edgesPerDay = edgesPerDay; + } + + public long getDatapointsPerDay() { + return datapointsPerDay; + } + + public void setDatapointsPerDay(long datapointsPerDay) { + this.datapointsPerDay = datapointsPerDay; + } + + public long getIngestBytesPerDay() { + return ingestBytesPerDay; + } + + public void setIngestBytesPerDay(long ingestBytesPerDay) { + this.ingestBytesPerDay = ingestBytesPerDay; + } + } + + /** + * Lifetime ceilings: how large a tenant may grow, rather than how fast. These are the free + * playground's dimensions, since a public signup lands in one; a paying tenant's + * {@code tenant_limits} row sets them to 0. + */ + public static class Lifetime { + + /** + * Off unless a deployment asks for it. These numbers size a free playground, and switching + * them on applies them to every tenant without an override, so one already holding more than + * {@code maxResources} would start refusing writes at the next restart. + */ + private boolean enabled = false; + + private long maxResources = 1_000; + private long maxEventsTotal = 25_000; + private long maxDatapointsTotal = 1_000_000_000; + private long maxTextDatapointsTotal = 100_000; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public long getMaxResources() { + return maxResources; + } + + public void setMaxResources(long maxResources) { + this.maxResources = maxResources; + } + + public long getMaxEventsTotal() { + return maxEventsTotal; + } + + public void setMaxEventsTotal(long maxEventsTotal) { + this.maxEventsTotal = maxEventsTotal; + } + + public long getMaxDatapointsTotal() { + return maxDatapointsTotal; + } + + public void setMaxDatapointsTotal(long maxDatapointsTotal) { + this.maxDatapointsTotal = maxDatapointsTotal; + } + + public long getMaxTextDatapointsTotal() { + return maxTextDatapointsTotal; + } + + public void setMaxTextDatapointsTotal(long maxTextDatapointsTotal) { + this.maxTextDatapointsTotal = maxTextDatapointsTotal; + } + } + + /** + * Requests per minute, per tenant and per user. Deployment-wide defaults; a tenant's + * {@code tenant_limits} row overrides any of them. 0 or negative disables that budget. + */ + public static class Rate { + + private boolean enabled = true; + + /** + * The primary budget: a public signup gets an organization of its own, so a tenant is a + * customer. The per-user figures are the backstop that keeps one identity inside a tenant + * from spending the whole tenant's allowance. + */ + private int writePerMinutePerTenant = 2_000; + private int readPerMinutePerTenant = 6_000; + private int writePerMinutePerUser = 600; + private int readPerMinutePerUser = 1_200; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public int getWritePerMinutePerTenant() { + return writePerMinutePerTenant; + } + + public void setWritePerMinutePerTenant(int writePerMinutePerTenant) { + this.writePerMinutePerTenant = writePerMinutePerTenant; + } + + public int getReadPerMinutePerTenant() { + return readPerMinutePerTenant; + } + + public void setReadPerMinutePerTenant(int readPerMinutePerTenant) { + this.readPerMinutePerTenant = readPerMinutePerTenant; + } + + public int getWritePerMinutePerUser() { + return writePerMinutePerUser; + } + + public void setWritePerMinutePerUser(int writePerMinutePerUser) { + this.writePerMinutePerUser = writePerMinutePerUser; + } + + public int getReadPerMinutePerUser() { + return readPerMinutePerUser; + } + + public void setReadPerMinutePerUser(int readPerMinutePerUser) { + this.readPerMinutePerUser = readPerMinutePerUser; + } + } +} diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/EdgeController.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/EdgeController.java index 2e848ef3..29790ab5 100644 --- a/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/EdgeController.java +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/EdgeController.java @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later package ai.intellistream.datahub.api.controllers; +import ai.intellistream.datahub.api.controllers.errors.LimitException; import ai.intellistream.datahub.api.controllers.errors.BadRequestError; import ai.intellistream.datahub.api.controllers.errors.BadRequestException; import ai.intellistream.datahub.api.controllers.errors.ConflictError; @@ -241,6 +242,10 @@ public ResponseEntity create( // Let dataset-ACL denials surface as 403 instead of being masked as 500 below. catch (AccessDeniedException e){ throw e; + } catch (LimitException e){ + // A limit refusal is an answer, not a fault: without this the catch below + // flattens it into a 500 and the caller never learns which limit they hit. + throw e; } catch (PulsarClientException | RuntimeException e){ log.error(e.getMessage(), e); return ResponseEntity.internalServerError().build(); diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/EventController.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/EventController.java index 8a3cafe1..398ee439 100644 --- a/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/EventController.java +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/EventController.java @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later package ai.intellistream.datahub.api.controllers; +import ai.intellistream.datahub.api.controllers.errors.LimitException; import ai.intellistream.datahub.api.controllers.errors.*; import ai.intellistream.datahub.api.responses.DataWrapper; import ai.intellistream.datahub.api.responses.swaggerdto.EventCountResponse; @@ -395,6 +396,11 @@ public ResponseEntity create( catch (org.springframework.security.access.AccessDeniedException e){ throw e; } + catch (LimitException e){ + // A limit refusal is an answer, not a fault: without this the catch below + // flattens it into a 500 and the caller never learns which limit they hit. + throw e; + } catch (PulsarClientException | RuntimeException e){ log.error(e.getMessage(), e); return ResponseEntity.internalServerError().build(); @@ -488,6 +494,11 @@ public ResponseEntity update( catch (org.springframework.security.access.AccessDeniedException e){ throw e; } + catch (LimitException e){ + // A limit refusal is an answer, not a fault: without this the catch below + // flattens it into a 500 and the caller never learns which limit they hit. + throw e; + } catch (PulsarClientException | RuntimeException e){ log.error(e.getMessage(), e); return ResponseEntity.internalServerError().build(); diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/FunctionController.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/FunctionController.java index fad73d2b..37e906ce 100644 --- a/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/FunctionController.java +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/FunctionController.java @@ -90,6 +90,10 @@ public ResponseEntity createFunction( return new ResponseEntity<>(error, HttpStatusCode.valueOf(error.getError().getCode())); } catch (org.springframework.security.access.AccessDeniedException e) { throw e; + } catch (LimitException e) { + // A limit refusal is an answer, not a fault: without this the catch below + // flattens it into a 500 and the caller never learns which limit they hit. + throw e; } catch (PulsarClientException | RuntimeException e) { log.error("Function create failed: {}", e.getMessage(), e); return ResponseEntity.internalServerError().build(); @@ -161,6 +165,10 @@ public ResponseEntity updateFunction( throw olf; } catch (org.springframework.security.access.AccessDeniedException e) { throw e; + } catch (LimitException e) { + // A limit refusal is an answer, not a fault: without this the catch below + // flattens it into a 500 and the caller never learns which limit they hit. + throw e; } catch (PulsarClientException | RuntimeException e) { log.error("Function update failed: {}", e.getMessage(), e); return ResponseEntity.internalServerError().build(); diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/ResourceController.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/ResourceController.java index 6b58329d..94e43f97 100644 --- a/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/ResourceController.java +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/ResourceController.java @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later package ai.intellistream.datahub.api.controllers; +import ai.intellistream.datahub.api.controllers.errors.LimitException; import ai.intellistream.datahub.api.policy.NamingPolicyViolationException; import ai.intellistream.datahub.api.controllers.errors.*; import ai.intellistream.datahub.api.responses.DataWrapper; @@ -149,7 +150,8 @@ public ResponseEntity fetchRelatedResources(@RequestBody RelatedResourcesForm @Operation( summary = "Find the nearest resources of a given label", description = """ - Breadth-first from a starting resource (numeric `id`), return the closest `limit` + Breadth-first from a starting resource (`id` or `externalId`), return the + closest `limit` nodes carrying one of `endLabels` (e.g. `["TIMESERIES"]`) plus the sub-graph that connects them. The cap is on matching END-nodes, not on hop depth or total node count — so "the 10 nearest time series" is exact however many intermediate nodes @@ -168,7 +170,9 @@ public ResponseEntity fetchRelatedResources(@RequestBody RelatedResourcesForm mediaType = MediaType.APPLICATION_JSON_VALUE, schema = @Schema(implementation = ResourceNetwork.class) )) - @ApiResponse(responseCode = "404", description = "The starting resource was not found. Check `id` and your tenant.", + @ApiResponse(responseCode = "404", + description = "The starting resource was not found. Check `id` / `externalId` " + + "and your tenant.", content = @Content( mediaType = MediaType.APPLICATION_JSON_VALUE, schema = @Schema(type = "string", example = "Could not find resource with id: 42") @@ -176,9 +180,7 @@ public ResponseEntity fetchRelatedResources(@RequestBody RelatedResourcesForm @RequestMapping(value = "/fetch-nearest", method = RequestMethod.POST, produces = { "application/json", "application/xml" }) public ResponseEntity fetchNearestResources(@RequestBody FetchNearestResourcesForm form) { try{ - ResourceNetwork network = resourceService.fetchNearestRelatedResources( - form.getId(), form.getEndLabels(), form.getLimit(), - form.getRelationshipTypes(), form.getExcludedLabels()); + ResourceNetwork network = resourceService.fetchNearestRelatedResources(form); return new ResponseEntity<>(network, HttpStatus.OK); } catch (ai.intellistream.datahub.errors.ObjectNotFoundException e){ // Rethrow so ObjectNotFoundExceptionHandler renders the shared RFC 9457 @@ -605,6 +607,11 @@ public ResponseEntity create( catch (org.springframework.security.access.AccessDeniedException e){ throw e; } + catch (LimitException e){ + // A limit refusal is an answer, not a fault: without this the catch below + // flattens it into a 500 and the caller never learns which limit they hit. + throw e; + } catch (PulsarClientException | RuntimeException e){ log.error(e.getMessage(), e); return ResponseEntity.internalServerError().build(); @@ -750,6 +757,11 @@ public ResponseEntity update( catch (org.springframework.security.access.AccessDeniedException e){ throw e; } + catch (LimitException e){ + // A limit refusal is an answer, not a fault: without this the catch below + // flattens it into a 500 and the caller never learns which limit they hit. + throw e; + } catch (DuplicateDataException e){ // A rename onto an external id already in use: the shared guard's 409, with the // offending ids, rather than the generic 500 the catch below would give. diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/TimeseriesController.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/TimeseriesController.java index dca9d69d..e483b91c 100644 --- a/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/TimeseriesController.java +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/TimeseriesController.java @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later package ai.intellistream.datahub.api.controllers; +import ai.intellistream.datahub.api.controllers.errors.LimitException; import ai.intellistream.datahub.api.policy.NamingPolicyViolationException; import ai.intellistream.datahub.api.controllers.errors.BadRequestError; import ai.intellistream.datahub.api.controllers.errors.BadRequestException; @@ -444,6 +445,11 @@ public ResponseEntity create( } catch (AccessDeniedException e){ throw e; } + catch (LimitException e){ + // A limit refusal is an answer, not a fault: without this the catch below + // flattens it into a 500 and the caller never learns which limit they hit. + throw e; + } catch (RuntimeException e){ log.error(e.getMessage(), e); return ResponseEntity.internalServerError().build(); @@ -551,6 +557,10 @@ public ResponseEntity update( // RuntimeException catch below would otherwise mask it as a 500. catch (OptimisticLockingFailureException olf) { throw olf; + } catch (LimitException e){ + // A limit refusal is an answer, not a fault: without this the catch below + // flattens it into a 500 and the caller never learns which limit they hit. + throw e; } catch (PulsarClientException | RuntimeException e){ log.error(e.getMessage(), e); return ResponseEntity.internalServerError().build(); @@ -826,7 +836,9 @@ public ResponseEntity insertDataPoints( // rest were inserted — report the misses with 404 and the per-entry error body. return new ResponseEntity<>(data, HttpStatus.NOT_FOUND); } - } catch (AccessDeniedException e){ + } catch (AccessDeniedException | LimitException e){ + // A limit refusal is an answer, not a fault: let it reach its advice, which turns it + // into the 429 or 403 that says which limit and how it clears. throw e; } catch (Exception e){ log.error(e.getMessage(), e); diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/errors/IngestQuotaExceededException.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/errors/IngestQuotaExceededException.java new file mode 100644 index 00000000..58148d37 --- /dev/null +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/errors/IngestQuotaExceededException.java @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.controllers.errors; + +/** + * The tenant has spent its ingest allowance for the current UTC day. + * + *

Temporary by construction: the window rolls at midnight, so this is a 429 with a + * {@code Retry-After} pointing there rather than a permanent refusal. + */ +public class IngestQuotaExceededException extends LimitException { + + private final String metric; + private final long limit; + private final long retryAfterSeconds; + + public IngestQuotaExceededException(String metric, long limit, long retryAfterSeconds) { + super("Daily %s ingest quota (%d) is spent; it resets at 00:00 UTC.".formatted(metric, limit)); + this.metric = metric; + this.limit = limit; + this.retryAfterSeconds = retryAfterSeconds; + } + + public String getMetric() { + return metric; + } + + public long getLimit() { + return limit; + } + + public long getRetryAfterSeconds() { + return retryAfterSeconds; + } +} diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/errors/LimitException.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/errors/LimitException.java new file mode 100644 index 00000000..fecab896 --- /dev/null +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/errors/LimitException.java @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.controllers.errors; + +/** + * A refusal because the tenant has reached a limit, rather than because the request was wrong. + * + *

The common base exists so the write endpoints can let both kinds past their terminal + * {@code catch (RuntimeException) -> 500} with a single rethrow. Without it a quota refusal reaches + * the caller as an unexplained 500 and reads as a platform fault instead of an answer. + */ +public abstract class LimitException extends RuntimeException { + + private final String detail; + + protected LimitException(String detail) { + super(detail); + this.detail = detail; + } + + /** + * The sentence the caller is shown. The same text as the exception message, kept as its own + * field because it is composed here from the metric and the number, never from anything a + * request or a lower layer supplied: it is an answer, not an error report. + */ + public String detail() { + return detail; + } +} diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/errors/LimitExceptionHandler.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/errors/LimitExceptionHandler.java new file mode 100644 index 00000000..299662c5 --- /dev/null +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/errors/LimitExceptionHandler.java @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.controllers.errors; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ProblemDetail; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +import java.net.URI; + +/** + * Turns the two limit refusals into RFC 9457 responses, with the status carrying the difference + * between them: a daily quota clears on its own, a lifetime ceiling does not. + */ +@RestControllerAdvice +@Order(Ordered.HIGHEST_PRECEDENCE) +@Slf4j +public class LimitExceptionHandler { + + /** + * 429 with {@code Retry-After}: the allowance returns at midnight UTC. The Java + * SDK already treats 429 as retryable, so a client with buffering enabled spools the batch and + * replays it once the window has rolled. + */ + @ExceptionHandler(IngestQuotaExceededException.class) + public ResponseEntity handleQuotaExceeded(IngestQuotaExceededException ex) { + log.info("Daily {} quota reached (limit {})", ex.getMetric(), ex.getLimit()); + + ProblemDetail problem = ProblemDetail.forStatusAndDetail(HttpStatus.TOO_MANY_REQUESTS, ex.detail()); + problem.setTitle("Ingest quota exceeded"); + problem.setType(URI.create("https://intellistream.ai/errors/ingest-quota-exceeded")); + problem.setProperty("metric", ex.getMetric()); + problem.setProperty("limit", ex.getLimit()); + problem.setProperty("retryAfter", ex.getRetryAfterSeconds()); + + return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS) + .header(HttpHeaders.RETRY_AFTER, String.valueOf(ex.getRetryAfterSeconds())) + .body(problem); + } + + /** + * 403, deliberately without {@code Retry-After}. Retrying will never succeed; + * the ceiling moves when someone raises it, which is what the message says. Matches how the + * files feature gate already answers a tenant it is switched off for. + */ + @ExceptionHandler(TenantLimitReachedException.class) + public ProblemDetail handleTenantLimitReached(TenantLimitReachedException ex) { + log.info("Tenant limit reached for {} (limit {})", ex.getMetric(), ex.getLimit()); + + ProblemDetail problem = ProblemDetail.forStatusAndDetail(HttpStatus.FORBIDDEN, ex.detail()); + problem.setTitle("Tenant limit reached"); + problem.setType(URI.create("https://intellistream.ai/errors/tenant-limit-reached")); + problem.setProperty("metric", ex.getMetric()); + problem.setProperty("limit", ex.getLimit()); + return problem; + } +} diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/errors/TenantLimitReachedException.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/errors/TenantLimitReachedException.java new file mode 100644 index 00000000..1bac0ec9 --- /dev/null +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/controllers/errors/TenantLimitReachedException.java @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.controllers.errors; + +/** + * The tenant has reached a lifetime ceiling: the size of its sandbox, not a rate. + * + *

Waiting does not clear this, so it is a 403 with no {@code Retry-After} and a message saying + * how it is lifted. The distinction matters to a client: the SDK retries a 429 and surfaces a 403, + * which is the right treatment for each. + */ +public class TenantLimitReachedException extends LimitException { + + private final String metric; + private final long limit; + + public TenantLimitReachedException(String metric, long limit) { + super(("This tenant has reached its limit of %d %s. Contact IntelliStream to have it " + + "raised.").formatted(limit, metric)); + this.metric = metric; + this.limit = limit; + } + + public String getMetric() { + return metric; + } + + public long getLimit() { + return limit; + } +} diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/filters/CachingBodyFilter.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/filters/CachingBodyFilter.java index 3cef8e9e..f9795d1c 100644 --- a/datahub-api/src/main/java/ai/intellistream/datahub/api/filters/CachingBodyFilter.java +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/filters/CachingBodyFilter.java @@ -24,7 +24,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha // Content-Length above 2 GB (Integer.MAX_VALUE); the upload request body is large and we // parse it straight off the raw stream. ReqLogService only logs bodies when these wrappers // are present, so passing the raw request/response through simply skips body logging here. - if (isStreamingFileEndpoint(httpRequest)) { + if (StreamingFileEndpoints.matches(httpRequest)) { try { chain.doFilter(request, response); } catch (IOException | ServletException e) { @@ -43,22 +43,4 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha } } - /** - * True for the file streaming endpoints whose bodies must not be buffered in memory: the file - * download ({@code GET /files/download/**}) and the file upload ({@code PUT /files}). - */ - private static boolean isStreamingFileEndpoint(HttpServletRequest request) { - String uri = request.getRequestURI(); - if (uri == null) { - return false; - } - String contextPath = request.getContextPath(); - if (contextPath != null && !contextPath.isEmpty() && uri.startsWith(contextPath)) { - uri = uri.substring(contextPath.length()); - } - if (uri.startsWith("/files/download/")) { - return true; - } - return "PUT".equalsIgnoreCase(request.getMethod()) && (uri.equals("/files") || uri.equals("/files/")); - } } diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/filters/RateLimitFilter.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/filters/RateLimitFilter.java new file mode 100644 index 00000000..902af946 --- /dev/null +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/filters/RateLimitFilter.java @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.filters; + +import ai.intellistream.datahub.api.config.LimitsProperties; +import ai.intellistream.datahub.api.services.TenantLimits; +import ai.intellistream.datahub.api.services.TenantLimitsService; +import ai.intellistream.datahub.services.ValkeyService; +import ai.intellistream.datahub.tenant.TenantContext; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.oauth2.jwt.Jwt; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.IOException; +import java.time.Instant; +import java.util.Set; + +/** + * Caps how many requests one tenant, and one user inside it, may make per minute. + * + *

Sits after authentication and before tenant provisioning, so an abusive caller is turned away + * before the request costs a Vault lookup or a Flyway check, and so {@code /mcp/*} is covered by the + * same rule as REST — the tools reach the services directly, but they arrive through this chain. + * + *

The window is a fixed minute rather than a token bucket. One atomic increment per request is + * correct across instances with no shared state to reconcile, where a bucket would need a + * read-modify-write of stored state per call to buy a smoothness that abuse protection does not + * need: the worst case here is a caller who spends two windows' allowance across a window boundary, + * which is still bounded. + * + *

A Valkey failure lets the request through. A limiter that cannot count is not a reason to + * refuse traffic that is otherwise legitimate, and the ceilings below it (body size, batch size, + * per-entity caps) still apply. + */ +@Slf4j +public class RateLimitFilter extends OncePerRequestFilter { + + /** Long enough that a window's key outlives the window even with clock skew between instances. */ + private static final long WINDOW_KEY_TTL_SECONDS = 120; + + private final LimitsProperties limits; + private final TenantLimitsService tenantLimits; + private final ValkeyService valkeyService; + + public RateLimitFilter(LimitsProperties limits, + TenantLimitsService tenantLimits, + ValkeyService valkeyService) { + this.limits = limits; + this.tenantLimits = tenantLimits; + this.valkeyService = valkeyService; + } + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) + throws ServletException, IOException { + + if (!limits.getRate().isEnabled()) { + chain.doFilter(request, response); + return; + } + + String tenantId = TenantContext.getTenantId(); + if (tenantId == null) { + // A permit-all endpoint (session, swagger, the live-tail handshake). Those are bounded + // per IP at the edge; there is no identity here to charge a request to. + chain.doFilter(request, response); + return; + } + + TenantLimits effective = tenantLimits.forTenant(tenantId); + boolean write = isWrite(request); + long epochMinute = Instant.now().getEpochSecond() / 60; + + int tenantLimit = write ? effective.writePerMinutePerTenant() : effective.readPerMinutePerTenant(); + int userLimit = write ? effective.writePerMinutePerUser() : effective.readPerMinutePerUser(); + String kind = write ? "w" : "r"; + + try { + if (over(tenantLimit, "dh:rl:t:%s:%s:%d".formatted(tenantId, kind, epochMinute))) { + refuse(request, response, tenantLimit, "tenant"); + return; + } + String subject = currentSubject(); + if (subject != null + && over(userLimit, "dh:rl:u:%s:%s:%d".formatted(subject, kind, epochMinute))) { + refuse(request, response, userLimit, "user"); + return; + } + } catch (RuntimeException e) { + log.warn("Rate limiting unavailable ({}); allowing the request through.", e.toString()); + } + + chain.doFilter(request, response); + } + + /** Counts this request, and reports whether it has taken the caller past {@code limit}. */ + private boolean over(int limit, String key) { + if (TenantLimits.unlimited(limit)) { + return false; + } + return valkeyService.incrementAndExpireIfNew(key, 1, WINDOW_KEY_TTL_SECONDS) > limit; + } + + /** + * The last path segment of the endpoints that POST only because they carry a filter body. They + * read, so they belong on the read budget: charging a console user's browsing to the smaller + * write allowance would throttle looking at data long before anyone wrote any. + */ + private static final Set READ_SHAPED_POST_SEGMENTS = Set.of( + "filter", "search", "byids", "list", "count", "check", + "fetch-related", "fetch-nearest", "aggregate", "latest"); + + private static boolean isWrite(HttpServletRequest request) { + String method = request.getMethod(); + if ("GET".equalsIgnoreCase(method) || "HEAD".equalsIgnoreCase(method) + || "OPTIONS".equalsIgnoreCase(method)) { + return false; + } + if ("POST".equalsIgnoreCase(method) && isReadShaped(request)) { + return false; + } + return true; + } + + private static boolean isReadShaped(HttpServletRequest request) { + String uri = request.getRequestURI(); + if (uri == null) { + return false; + } + int lastSlash = uri.lastIndexOf('/'); + String segment = lastSlash < 0 ? uri : uri.substring(lastSlash + 1); + return READ_SHAPED_POST_SEGMENTS.contains(segment.toLowerCase()); + } + + /** The JWT {@code sub}, or null when there is no authenticated principal. */ + private static String currentSubject() { + Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); + if (authentication == null) { + return null; + } + if (authentication.getPrincipal() instanceof Jwt jwt) { + return jwt.getSubject(); + } + return authentication.getName(); + } + + /** + * Written straight to the response: a filter runs outside the {@code @RestControllerAdvice} + * chain, so nothing downstream would shape this body. + */ + private static void refuse(HttpServletRequest request, HttpServletResponse response, int limit, String scope) + throws IOException { + long retryAfter = 60 - (Instant.now().getEpochSecond() % 60); + log.info("Rate limit reached ({} scope, {}/min) on {} {}", scope, limit, + request.getMethod(), request.getRequestURI()); + + if (response.isCommitted()) { + return; + } + response.reset(); + response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); + response.setHeader(HttpHeaders.RETRY_AFTER, String.valueOf(retryAfter)); + response.setContentType(MediaType.APPLICATION_PROBLEM_JSON_VALUE); + response.setCharacterEncoding("UTF-8"); + response.getWriter().write(""" + {"type":"https://intellistream.ai/errors/rate-limit-exceeded",\ + "title":"Too many requests",\ + "status":429,\ + "detail":"This %s has used its %d requests per minute. Retry in %d seconds.",\ + "scope":"%s",\ + "limit":%d,\ + "retryAfter":%d}""" + .formatted(scope, limit, retryAfter, scope, limit, retryAfter)); + response.getWriter().flush(); + } +} diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/filters/RequestBodySizeLimitFilter.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/filters/RequestBodySizeLimitFilter.java new file mode 100644 index 00000000..409d429e --- /dev/null +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/filters/RequestBodySizeLimitFilter.java @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.filters; + +import ai.intellistream.datahub.api.config.LimitsProperties; +import ai.intellistream.datahub.api.controllers.errors.IngestQuotaExceededException; +import ai.intellistream.datahub.api.services.IngestQuotaService; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ReadListener; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletInputStream; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequestWrapper; +import jakarta.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.IOException; + +/** + * Rejects a request body larger than the configured cap. + * + *

The api is not only reached through nginx — anything that can talk to the service port sends + * whatever it likes — so the ceiling has to exist in the application. {@code Content-Length} is + * checked before the body is read; a chunked body with no declared length is counted as it is + * consumed, so neither shape can get past the cap. + * + *

413 rather than 429 is deliberate: the size of a request never becomes acceptable by waiting, + * and the Java SDK retries 429/5xx while surfacing 4xx to the caller. + */ +@Slf4j +public class RequestBodySizeLimitFilter extends OncePerRequestFilter { + + private static final String DATAPOINT_INSERT_PATH = "/timeseries/data"; + + private final LimitsProperties limits; + private final IngestQuotaService ingestQuota; + + public RequestBodySizeLimitFilter(LimitsProperties limits, IngestQuotaService ingestQuota) { + this.limits = limits; + this.ingestQuota = ingestQuota; + } + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) + throws ServletException, IOException { + + // The upload streams to disk and the download has no request body worth counting. + if (StreamingFileEndpoints.matches(request)) { + chain.doFilter(request, response); + return; + } + + long limit = limitFor(request); + if (limit <= 0) { + chain.doFilter(request, response); + return; + } + + long declared = request.getContentLengthLong(); + if (declared > limit) { + reject(request, response, limit, declared); + return; + } + + // Bytes are charged here because this is where a body's size is known. It is the quota that + // actually bounds storage growth: an entity count does not, since one legal entity may be a + // few hundred KB. Over the daily allowance surfaces as the same 429 a controller would give. + if (declared > 0 && isWrite(request.getMethod())) { + try { + ingestQuota.checkAndRecord(IngestQuotaService.QuotaMetric.BYTES, declared); + } catch (IngestQuotaExceededException e) { + refuseOverQuota(response, e); + return; + } + } + + chain.doFilter(new CountingRequestWrapper(request, limit), response); + } + + private static boolean isWrite(String method) { + return "POST".equalsIgnoreCase(method) + || "PUT".equalsIgnoreCase(method) + || "PATCH".equalsIgnoreCase(method) + || "DELETE".equalsIgnoreCase(method); + } + + /** Same 429 the advice would produce, written here because a filter never reaches one. */ + private static void refuseOverQuota(HttpServletResponse response, IngestQuotaExceededException e) + throws IOException { + if (response.isCommitted()) { + return; + } + response.reset(); + response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); + response.setHeader(HttpHeaders.RETRY_AFTER, String.valueOf(e.getRetryAfterSeconds())); + response.setContentType(MediaType.APPLICATION_PROBLEM_JSON_VALUE); + response.setCharacterEncoding("UTF-8"); + response.getWriter().write(""" + {"type":"https://intellistream.ai/errors/ingest-quota-exceeded",\ + "title":"Ingest quota exceeded",\ + "status":429,\ + "detail":"%s",\ + "metric":"%s",\ + "limit":%d,\ + "retryAfter":%d}""" + .formatted(e.detail(), e.getMetric(), e.getLimit(), e.getRetryAfterSeconds())); + response.getWriter().flush(); + } + + /** Datapoint inserts get their own, larger ceiling; everything else shares the general one. */ + private long limitFor(HttpServletRequest request) { + String uri = request.getRequestURI(); + if (uri == null) { + return limits.getMaxBodyBytes(); + } + String contextPath = request.getContextPath(); + if (contextPath != null && !contextPath.isEmpty() && uri.startsWith(contextPath)) { + uri = uri.substring(contextPath.length()); + } + if (uri.equals(DATAPOINT_INSERT_PATH) || uri.equals(DATAPOINT_INSERT_PATH + "/")) { + return limits.getMaxBodyBytesDatapoints(); + } + return limits.getMaxBodyBytes(); + } + + /** + * Written here rather than raised as an exception: a filter sits outside the + * {@code @RestControllerAdvice} chain, so nothing downstream would shape the body. + */ + private static void reject(HttpServletRequest request, HttpServletResponse response, long limit, long actual) + throws IOException { + log.warn("Rejecting oversized request body on {}: {} bytes, limit {}", + request.getRequestURI(), actual < 0 ? "unknown" : actual, limit); + if (response.isCommitted()) { + return; + } + response.reset(); + response.setStatus(HttpStatus.PAYLOAD_TOO_LARGE.value()); + response.setContentType(MediaType.APPLICATION_PROBLEM_JSON_VALUE); + response.setCharacterEncoding("UTF-8"); + response.getWriter().write(""" + {"type":"https://intellistream.ai/errors/request-too-large",\ + "title":"Request body too large",\ + "status":413,\ + "detail":"The request body exceeds the %d byte limit for this endpoint.",\ + "limitBytes":%d}""" + .formatted(limit, limit)); + response.getWriter().flush(); + } + + /** Fails the read as soon as more than {@code limit} bytes have been consumed. */ + private static final class CountingRequestWrapper extends HttpServletRequestWrapper { + + private final long limit; + + private CountingRequestWrapper(HttpServletRequest request, long limit) { + super(request); + this.limit = limit; + } + + @Override + public ServletInputStream getInputStream() throws IOException { + return new CountingServletInputStream(super.getInputStream(), limit); + } + } + + private static final class CountingServletInputStream extends ServletInputStream { + + private final ServletInputStream delegate; + private final long limit; + private long count; + + private CountingServletInputStream(ServletInputStream delegate, long limit) { + this.delegate = delegate; + this.limit = limit; + } + + @Override + public int read() throws IOException { + int b = delegate.read(); + if (b != -1) { + add(1); + } + return b; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int read = delegate.read(b, off, len); + if (read > 0) { + add(read); + } + return read; + } + + private void add(int read) throws IOException { + count += read; + if (count > limit) { + throw new RequestBodyTooLargeException(limit); + } + } + + @Override + public boolean isFinished() { + return delegate.isFinished(); + } + + @Override + public boolean isReady() { + return delegate.isReady(); + } + + @Override + public void setReadListener(ReadListener readListener) { + delegate.setReadListener(readListener); + } + + @Override + public int available() throws IOException { + return delegate.available(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + } + + /** + * Thrown mid-read for a body with no usable {@code Content-Length}. It surfaces as an unreadable + * request body, which the api already answers with a 400 — the right class of answer, and the + * only one still available once the response has started. + */ + public static class RequestBodyTooLargeException extends IOException { + public RequestBodyTooLargeException(long limit) { + super("Request body exceeds the " + limit + " byte limit."); + } + } +} diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/filters/StreamingFileEndpoints.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/filters/StreamingFileEndpoints.java new file mode 100644 index 00000000..c375e2c4 --- /dev/null +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/filters/StreamingFileEndpoints.java @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.filters; + +import jakarta.servlet.http.HttpServletRequest; + +/** + * The file endpoints whose bodies stream and must never be buffered or counted: the download + * ({@code GET /files/download/**}) and the upload ({@code PUT /files}). Shared so the body-cache + * filter and the body-size cap cannot disagree about which requests are exempt. + */ +public final class StreamingFileEndpoints { + + private StreamingFileEndpoints() { + } + + public static boolean matches(HttpServletRequest request) { + String uri = request.getRequestURI(); + if (uri == null) { + return false; + } + String contextPath = request.getContextPath(); + if (contextPath != null && !contextPath.isEmpty() && uri.startsWith(contextPath)) { + uri = uri.substring(contextPath.length()); + } + if (uri.startsWith("/files/download/")) { + return true; + } + return "PUT".equalsIgnoreCase(request.getMethod()) && (uri.equals("/files") || uri.equals("/files/")); + } +} diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/services/EventService.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/services/EventService.java index 32bfd062..73a4a841 100644 --- a/datahub-api/src/main/java/ai/intellistream/datahub/api/services/EventService.java +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/services/EventService.java @@ -65,6 +65,7 @@ public class EventService { private final DataSetRepository dataSetRepository; /** The one authority for "which data sets are beneath this one" — shared with the ACL. */ private final DatasetClosureService datasetClosureService; + private final IngestQuotaService ingestQuota; /** * The dataset ids the caller may read, or {@code null} when the caller may read every dataset @@ -312,6 +313,9 @@ public DataWrapper create(DataWrapper apiReqData) return dw; } + // Charged here rather than in a filter: event_create reaches this method directly. + ingestQuota.checkAndRecord(IngestQuotaService.QuotaMetric.EVENTS, eventModels.size()); + Set dataSets = new HashSet<>(); Set externalIdList = new HashSet<>(); Set resourceIds = new HashSet<>(); diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/services/IngestQuotaService.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/services/IngestQuotaService.java new file mode 100644 index 00000000..fb516682 --- /dev/null +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/services/IngestQuotaService.java @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.services; + +import ai.intellistream.datahub.api.controllers.errors.IngestQuotaExceededException; +import ai.intellistream.datahub.api.controllers.errors.TenantLimitReachedException; +import ai.intellistream.datahub.services.ValkeyService; +import ai.intellistream.datahub.tenant.TenantContext; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +/** + * How much a tenant may ingest: a rolling daily allowance, and a lifetime ceiling. + * + *

The two answer different questions. The daily quota is a rate — spend it and it returns at + * midnight — and is what stops a caller turning steady, individually legal requests into bulk + * storage. The lifetime ceiling is the size of the sandbox a free tenant gets, and only moves when + * someone raises it. + * + *

Counted the way {@link LiveIngestCounter} counts: accumulated in memory per tenant and flushed + * to Valkey every couple of seconds, so a per-request check costs no round trip. The cost is that a + * tenant can overshoot by whatever the other instances ingest inside one flush interval. That is the + * right trade here — these are ceilings, not invoices — and the overshoot is bounded by the interval + * rather than growing with traffic. + * + *

Enforced in the service layer rather than in a filter, because the MCP tools call the services + * directly and would otherwise be uncounted. Bytes are the exception: they are charged by the + * request-size filter, which is where the size of a body is known. + * + *

A Valkey failure lets ingest through. Refusing writes because a counter is unreachable would + * turn a cache outage into an ingest outage. + */ +@Slf4j +@Service +public class IngestQuotaService { + + private static final long FLUSH_MS = 2000; + + /** Two days, so a window's key outlives the window even with clock skew between instances. */ + private static final long DAILY_KEY_TTL_SECONDS = 2 * 24 * 3600; + + private static final DateTimeFormatter DAY = DateTimeFormatter.ofPattern("yyyyMMdd").withZone(ZoneOffset.UTC); + + /** + * What is counted. Nodes share one budget because resources, timeseries, datasets, labels, + * policies and functions are all rows in the same table: separate budgets would be six doors + * into the same room. + */ + public enum QuotaMetric { + EVENTS("events"), + NODES("nodes"), + EDGES("relationships"), + DATAPOINTS("data points"), + /** Charged alongside DATAPOINTS for a TEXT/MIXED series, which is the expensive kind. */ + TEXT_DATAPOINTS("text data points"), + BYTES("ingested bytes"); + + private final String label; + + QuotaMetric(String label) { + this.label = label; + } + + public String label() { + return label; + } + + String key() { + return name().toLowerCase(); + } + } + + /** One Valkey counter. {@code daily} decides whether it expires and which limit it answers to. */ + private record Counter(String tenantId, String valkeyKey, boolean daily) { + } + + private final TenantLimitsService tenantLimits; + private final ValkeyService valkeyService; + private final Supplier clock; + + /** Counted but not yet flushed. */ + private final Map pending = new ConcurrentHashMap<>(); + + /** The last total Valkey reported, so a check costs no round trip of its own. */ + private final Map known = new ConcurrentHashMap<>(); + + // Explicit, because the test constructor below makes this an ambiguous choice otherwise. + @Autowired + public IngestQuotaService(TenantLimitsService tenantLimits, ValkeyService valkeyService) { + this(tenantLimits, valkeyService, Instant::now); + } + + /** Test seam: a clock that can be moved across a day boundary. */ + IngestQuotaService(TenantLimitsService tenantLimits, ValkeyService valkeyService, Supplier clock) { + this.tenantLimits = tenantLimits; + this.valkeyService = valkeyService; + this.clock = clock; + } + + /** + * Charge {@code count} to the current tenant, refusing if that would take it past either + * ceiling. Call after validation and authorization, so a request that was going to fail anyway + * does not spend the tenant's allowance. + * + * @throws IngestQuotaExceededException if the daily allowance is spent (429, retryable) + * @throws TenantLimitReachedException if the lifetime ceiling is reached (403, not retryable) + */ + public void checkAndRecord(QuotaMetric metric, long count) { + String tenantId = TenantContext.getTenantId(); + if (tenantId == null || count <= 0) { + return; + } + TenantLimits limits = tenantLimits.forTenant(tenantId); + + long dailyLimit = dailyLimit(limits, metric); + Counter daily = dailyCounter(tenantId, metric); + if (!TenantLimits.unlimited(dailyLimit) && used(daily) + count > dailyLimit) { + throw new IngestQuotaExceededException(metric.label(), dailyLimit, secondsUntilUtcMidnight()); + } + + long lifetimeLimit = lifetimeLimit(limits, metric); + Counter lifetime = lifetimeCounter(tenantId, metric); + if (!TenantLimits.unlimited(lifetimeLimit) && used(lifetime) + count > lifetimeLimit) { + throw new TenantLimitReachedException(metric.label(), lifetimeLimit); + } + + // Only what is actually bounded is counted: the daily counter is the rate, the lifetime one + // the running total, and a metric with neither limit set costs no Valkey traffic at all. + if (!TenantLimits.unlimited(dailyLimit)) { + add(daily, count); + } + if (!TenantLimits.unlimited(lifetimeLimit)) { + add(lifetime, count); + } + } + + /** The tenant's lifetime total for a metric, as far as this instance knows it. */ + public long lifetimeTotal(QuotaMetric metric) { + String tenantId = TenantContext.getTenantId(); + return tenantId == null ? 0 : used(lifetimeCounter(tenantId, metric)); + } + + private void add(Counter counter, long count) { + pending.computeIfAbsent(counter, k -> new AtomicLong()).addAndGet(count); + } + + /** + * What this tenant has spent: the last flushed total plus what this instance is holding. The + * local part matters — without it a burst inside one flush interval reads a stale total and + * sails past the ceiling. + */ + private long used(Counter counter) { + long flushed = known.computeIfAbsent(counter.valkeyKey(), this::readTotal); + AtomicLong local = pending.get(counter); + return flushed + (local == null ? 0 : local.get()); + } + + private long readTotal(String key) { + try { + String raw = valkeyService.getString(key); + return raw == null ? 0L : Long.parseLong(raw); + } catch (RuntimeException e) { + log.debug("Could not read quota counter {}: {}", key, e.toString()); + return 0L; + } + } + + /** + * Push what has accumulated to Valkey. The reply is the cluster-wide total and becomes what the + * next check reads, so instances counting the same tenant converge within an interval. + */ + @Scheduled(fixedRate = FLUSH_MS) + void flush() { + for (Map.Entry entry : pending.entrySet()) { + long delta = entry.getValue().getAndSet(0); + if (delta <= 0) { + continue; + } + Counter counter = entry.getKey(); + try { + TenantContext.runWith(counter.tenantId(), () -> { + long total = counter.daily() + ? valkeyService.incrementAndExpireIfNew(counter.valkeyKey(), delta, DAILY_KEY_TTL_SECONDS) + : valkeyService.increment(counter.valkeyKey(), delta); + known.put(counter.valkeyKey(), total); + }); + } catch (Exception e) { + log.warn("Quota flush failed for {}, retrying next interval: {}", + counter.valkeyKey(), e.getMessage()); + // Fold the delta back in rather than losing it to a transient Valkey failure. + entry.getValue().addAndGet(delta); + } + } + } + + private Counter dailyCounter(String tenantId, QuotaMetric metric) { + return new Counter(tenantId, + "dh:quota:%s:%s:%s".formatted(metric.key(), tenantId, DAY.format(clock.get())), + true); + } + + private Counter lifetimeCounter(String tenantId, QuotaMetric metric) { + return new Counter(tenantId, "dh:quota:total:%s:%s".formatted(metric.key(), tenantId), false); + } + + private long secondsUntilUtcMidnight() { + Instant now = clock.get(); + Instant midnight = now.atZone(ZoneOffset.UTC).toLocalDate().plusDays(1) + .atStartOfDay(ZoneOffset.UTC).toInstant(); + return Math.max(1, midnight.getEpochSecond() - now.getEpochSecond()); + } + + private static long dailyLimit(TenantLimits limits, QuotaMetric metric) { + return switch (metric) { + case EVENTS -> limits.eventsPerDay(); + case NODES -> limits.nodesPerDay(); + case EDGES -> limits.edgesPerDay(); + case DATAPOINTS -> limits.datapointsPerDay(); + case BYTES -> limits.ingestBytesPerDay(); + // Covered by the DATAPOINTS allowance; this metric exists for its lifetime ceiling. + case TEXT_DATAPOINTS -> 0; + }; + } + + private static long lifetimeLimit(TenantLimits limits, QuotaMetric metric) { + return switch (metric) { + case EVENTS -> limits.maxEventsTotal(); + case DATAPOINTS -> limits.maxDatapointsTotal(); + case TEXT_DATAPOINTS -> limits.maxTextDatapointsTotal(); + // Resources are counted live against max_resources, where a delete frees room. + case NODES, EDGES, BYTES -> 0; + }; + } +} diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/services/PolicyService.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/services/PolicyService.java index 19fcbf6e..95e4e91d 100644 --- a/datahub-api/src/main/java/ai/intellistream/datahub/api/services/PolicyService.java +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/services/PolicyService.java @@ -354,20 +354,7 @@ private static String requireNonBlank(String value, String field) { return value; } - /** - * Re-assert a policy node in the graph after a write, so an edit does not leave Neo4j stale. - * - *

Why the action is CREATE for what is logically an update. The Neo4j consumer's - * {@code UPDATE} branch reads only {@code updateResourceForms}/{@code updateTimeseries} and - * ignores {@code resources} entirely ({@code GraphEventNeo4jListener.updateResourceAndRelations}), - * and the policy layer does not build those label-oriented forms. Its {@code createResource}, - * by contrast, is an idempotent MERGE-on-id upsert that re-asserts every node property — so a - * CREATE matches the existing node rather than duplicating it, and actually applies the change. - * Sending UPDATE here would be more honest and would silently do nothing. - * - *

Fixing that properly means teaching the consumer's UPDATE path to accept full resources - * (or having policies build update forms); tracked in the audit backlog. - */ + /** Re-assert a policy node in the graph after a write, so an edit does not leave Neo4j stale. */ private void publishPolicyUpsert(PolicyEntity saved) { graphOutbox.queueUpsert(List.of(saved), List.of()); } diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/services/ResourceService.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/services/ResourceService.java index cedec2f8..bad2d6c2 100644 --- a/datahub-api/src/main/java/ai/intellistream/datahub/api/services/ResourceService.java +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/services/ResourceService.java @@ -9,6 +9,7 @@ import ai.intellistream.datahub.models.policy.PolicyFinding; import ai.intellistream.datahub.models.policy.PolicyWarning; import ai.intellistream.datahub.api.controllers.errors.BadRequestError; +import ai.intellistream.datahub.api.controllers.errors.TenantLimitReachedException; import ai.intellistream.datahub.api.controllers.errors.BadRequestException; import org.springframework.dao.OptimisticLockingFailureException; import ai.intellistream.datahub.api.controllers.errors.DuplicateDataException; @@ -109,6 +110,8 @@ public class ResourceService { /** The naming policy, applied to every create and update. See {@link PolicyEnforcement}. */ private final PolicyEnforcement policyEnforcement; + private final IngestQuotaService ingestQuota; + private final TenantLimitsService tenantLimitsService; /** The one authority for "which data sets are beneath this one" — shared with the ACL. */ private final DatasetClosureService datasetClosureService; @@ -137,9 +140,13 @@ public ResourceService( Validator validator, PolicyEnforcement policyEnforcement, DatasetClosureService datasetClosureService, + IngestQuotaService ingestQuota, + TenantLimitsService tenantLimitsService, EdgeMapper edgeMapper, NodeUpdateService nodeUpdateService, NamingPolicyResolver namingPolicyResolver){ + this.ingestQuota = ingestQuota; + this.tenantLimitsService = tenantLimitsService; this.entityManager = entityManager; this.nodeRepository = nodeRepository; this.nodeService = nodeService; @@ -159,6 +166,33 @@ public ResourceService( this.namingPolicyResolver = namingPolicyResolver; } + /** + * Refuse a batch that would take the tenant past its node ceiling. + * + *

Counted live rather than accumulated, so deleting frees room again — the friendly behaviour + * for a sandbox someone is experimenting in, and the reason this ceiling is not just another + * counter in {@link IngestQuotaService}. Only a capped tenant pays for the query, and a capped + * tenant is by definition one whose node table is small. + */ + private void assertRoomForMoreNodes(int incoming) { + TenantLimits limits = tenantLimitsService.current(); + // No answer means no ceiling, the same way an unreachable counter allows ingest: not knowing + // a limit is never a reason to refuse a write that is otherwise fine. + if (limits == null || incoming <= 0) { + return; + } + long limit = limits.maxResources(); + if (TenantLimits.unlimited(limit)) { + return; + } + if (nodeRepository.count() + incoming > limit) { + // Every node type lives in one table, so one ceiling covers them all. + throw new TenantLimitReachedException( + "objects (resources, time series, data sets, labels and policies share this limit)", + limit); + } + } + /** * Save assets to Postgres first, if everything goes well, submit assets * for further consumption in pulsar. Neo4j is a listener. @@ -180,6 +214,14 @@ public GraphDataWrapper create(GraphDataWrapper errors = new ResponseError<>(); @@ -1217,6 +1259,30 @@ public ResourceNetwork fetchRelatedResources(@Valid RelatedResourcesForm form) { form.getLimit(), form.getExcludedLabels()); } + /** + * Nearest-N from a start node given by either identifier. + * + *

{@link FetchNearestResourcesForm} declares {@code id} and {@code externalId} and its + * {@code @OneIdNotNull} validator accepts either, so a caller who sends only an external id has + * sent a valid request. Resolving it here is what makes that true in fact: without this the + * request reached {@code findById(null)} and failed as a 500, which reads as a server fault + * rather than as the perfectly good request it was. Mirrors {@link #fetchRelatedResources}, + * which has always accepted both. + */ + @Transactional(readOnly = true) + public ResourceNetwork fetchNearestRelatedResources(@Valid FetchNearestResourcesForm form) { + if (form.getId() == null) { + NodeEntity start = nodeRepository.findByExternalId(form.getExternalId()); + if (start == null) { + throw new ObjectNotFoundException( + "Resource with externalId: " + form.getExternalId() + " not found."); + } + form.setId(start.getId()); + } + return fetchNearestRelatedResources(form.getId(), form.getEndLabels(), form.getLimit(), + form.getRelationshipTypes(), form.getExcludedLabels()); + } + /** * ACL-gated variant of {@link #fetchRelatedResources} that returns the nearest {@code limit} nodes * carrying any of {@code endLabels} (breadth-first), via diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/services/TenantLimits.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/services/TenantLimits.java new file mode 100644 index 00000000..e0324a1f --- /dev/null +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/services/TenantLimits.java @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.services; + +/** + * The limits in force for one tenant: the deployment defaults with that tenant's overrides applied. + * + *

A value of 0 or below means unlimited, so a check reads the same way everywhere: + * {@code if (limit > 0 && used > limit)}. + */ +public record TenantLimits( + int writePerMinutePerTenant, + int readPerMinutePerTenant, + int writePerMinutePerUser, + int readPerMinutePerUser, + long eventsPerDay, + long nodesPerDay, + long edgesPerDay, + long datapointsPerDay, + long ingestBytesPerDay, + long maxResources, + long maxEventsTotal, + long maxDatapointsTotal, + long maxTextDatapointsTotal, + int maxWsSocketsPerTenant, + int maxWsSocketsPerUser) { + + public static boolean unlimited(long limit) { + return limit <= 0; + } +} diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/services/TenantLimitsService.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/services/TenantLimitsService.java new file mode 100644 index 00000000..e3a76eae --- /dev/null +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/services/TenantLimitsService.java @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.services; + +import ai.intellistream.datahub.api.config.LimitsProperties; +import ai.intellistream.datahub.tenant.TenantContext; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Service; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Duration; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.LongSupplier; + +/** + * The limits in force for the request's tenant: the {@code datahub.limits.*} defaults with that + * tenant's {@code tenant_limits} row applied over them. + * + *

Cached in-process for a few minutes, so a limit that is consulted on every request costs one + * query per tenant per instance per TTL rather than a round trip per call. The TTL is also the + * propagation delay: an operator raising a limit for a tenant that has hit it sees it take effect + * across every instance within that window, with no restart and no cache to invalidate by hand. + * + *

Any failure resolves to the defaults rather than propagating. This is consulted from a servlet + * filter that runs before tenant provisioning, so a first-touch request legitimately arrives before + * the schema exists — and a limits lookup that cannot answer is never a reason to refuse a request + * that is otherwise fine. + */ +@Slf4j +@Service +public class TenantLimitsService { + + private final LimitsProperties defaults; + private final JdbcTemplate jdbcTemplate; + private final long ttlMillis; + private final LongSupplier clock; + + private final ConcurrentMap cache = new ConcurrentHashMap<>(); + + private record Entry(TenantLimits limits, long resolvedAtMillis) { + } + + // Explicit, because the test constructor below makes this an ambiguous choice otherwise. + @Autowired + public TenantLimitsService(LimitsProperties defaults, + JdbcTemplate jdbcTemplate, + @Value("${datahub.limits.cache-ttl:5m}") Duration cacheTtl) { + this(defaults, jdbcTemplate, cacheTtl, System::currentTimeMillis); + } + + /** Test seam: a clock that can be advanced, so the cache TTL is assertable without sleeping. */ + TenantLimitsService(LimitsProperties defaults, + JdbcTemplate jdbcTemplate, + Duration cacheTtl, + LongSupplier clock) { + this.defaults = defaults; + this.jdbcTemplate = jdbcTemplate; + this.ttlMillis = cacheTtl.toMillis(); + this.clock = clock; + } + + /** The limits for the tenant on this thread, or the deployment defaults if there is none. */ + public TenantLimits current() { + String tenantId = TenantContext.getTenantId(); + if (tenantId == null) { + return fromDefaults(); + } + return forTenant(tenantId); + } + + public TenantLimits forTenant(String tenantId) { + long now = clock.getAsLong(); + Entry cached = cache.get(tenantId); + if (cached != null && now - cached.resolvedAtMillis() < ttlMillis) { + return cached.limits(); + } + TenantLimits resolved = load(tenantId); + cache.put(tenantId, new Entry(resolved, now)); + return resolved; + } + + private TenantLimits load(String tenantId) { + try { + TenantLimits row = jdbcTemplate.query( + "SELECT * FROM tenant_limits WHERE id = 1", + this::mapRow); + return row == null ? fromDefaults() : row; + } catch (RuntimeException e) { + // Includes the tenant whose schema is not provisioned yet: the rate-limit filter runs + // before provisioning, so this is expected on a first touch rather than exceptional. + log.debug("No tenant_limits for tenant {} ({}); using deployment defaults.", + tenantId, e.getClass().getSimpleName()); + return fromDefaults(); + } + } + + private TenantLimits mapRow(ResultSet rs) throws SQLException { + if (!rs.next()) { + return null; + } + TenantLimits base = fromDefaults(); + return new TenantLimits( + intOr(rs, "write_per_minute_per_tenant", base.writePerMinutePerTenant()), + intOr(rs, "read_per_minute_per_tenant", base.readPerMinutePerTenant()), + intOr(rs, "write_per_minute_per_user", base.writePerMinutePerUser()), + intOr(rs, "read_per_minute_per_user", base.readPerMinutePerUser()), + longOr(rs, "events_per_day", base.eventsPerDay()), + longOr(rs, "nodes_per_day", base.nodesPerDay()), + longOr(rs, "edges_per_day", base.edgesPerDay()), + longOr(rs, "datapoints_per_day", base.datapointsPerDay()), + longOr(rs, "ingest_bytes_per_day", base.ingestBytesPerDay()), + longOr(rs, "max_resources", base.maxResources()), + longOr(rs, "max_events_total", base.maxEventsTotal()), + longOr(rs, "max_datapoints_total", base.maxDatapointsTotal()), + longOr(rs, "max_text_datapoints_total", base.maxTextDatapointsTotal()), + intOr(rs, "max_ws_sockets_per_tenant", base.maxWsSocketsPerTenant()), + intOr(rs, "max_ws_sockets_per_user", base.maxWsSocketsPerUser())); + } + + /** A NULL column means "inherit"; the column being absent entirely means an older schema. */ + private static int intOr(ResultSet rs, String column, int fallback) throws SQLException { + int value = rs.getInt(column); + return rs.wasNull() ? fallback : value; + } + + private static long longOr(ResultSet rs, String column, long fallback) throws SQLException { + long value = rs.getLong(column); + return rs.wasNull() ? fallback : value; + } + + /** + * The deployment defaults. A whole section switched off (via its {@code enabled} flag) resolves + * to 0 across that section, which every check reads as unlimited. + */ + private TenantLimits fromDefaults() { + LimitsProperties.Rate rate = defaults.getRate(); + LimitsProperties.Quota quota = defaults.getQuota(); + LimitsProperties.Lifetime lifetime = defaults.getLifetime(); + LimitsProperties.WebSocket websocket = defaults.getWebsocket(); + boolean quotas = quota.isEnabled(); + boolean ceilings = lifetime.isEnabled(); + boolean sockets = websocket.isEnabled(); + return new TenantLimits( + rate.getWritePerMinutePerTenant(), + rate.getReadPerMinutePerTenant(), + rate.getWritePerMinutePerUser(), + rate.getReadPerMinutePerUser(), + quotas ? quota.getEventsPerDay() : 0, + quotas ? quota.getNodesPerDay() : 0, + quotas ? quota.getEdgesPerDay() : 0, + quotas ? quota.getDatapointsPerDay() : 0, + quotas ? quota.getIngestBytesPerDay() : 0, + ceilings ? lifetime.getMaxResources() : 0, + ceilings ? lifetime.getMaxEventsTotal() : 0, + ceilings ? lifetime.getMaxDatapointsTotal() : 0, + ceilings ? lifetime.getMaxTextDatapointsTotal() : 0, + sockets ? websocket.getMaxSocketsPerTenant() : 0, + sockets ? websocket.getMaxSocketsPerUser() : 0); + } +} diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/services/TimeseriesService.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/services/TimeseriesService.java index 9712287e..dfb0d1be 100644 --- a/datahub-api/src/main/java/ai/intellistream/datahub/api/services/TimeseriesService.java +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/services/TimeseriesService.java @@ -25,6 +25,7 @@ import ai.intellistream.datahub.clickhouse.DatapointBinaryConverter; import ai.intellistream.datahub.errors.ObjectNotFoundException; import ai.intellistream.datahub.errors.ResponseError; +import ai.intellistream.datahub.models.validation.FieldLimits; import ai.intellistream.datahub.helpers.datetime.DateTimeHandler; import ai.intellistream.datahub.jpa.domains.DatasetEntity; import ai.intellistream.datahub.jpa.domains.EdgeEntity; @@ -144,6 +145,8 @@ public class TimeseriesService { */ private final TransactionTemplate transactionTemplate; + private final IngestQuotaService ingestQuota; + // Bounded executor for ClickHouse datapoint queries. Replaces the previous `new Thread(...)` // per filter, which let any caller fan out unlimited threads. Sized off the host CPU count // because the work is I/O-bound (network + remote query execution) — the real limit is @@ -776,6 +779,13 @@ private void validateDataSet(Collection dataSetIds) { public DataWrapper insertDatapoints(DataWrapper data) throws PulsarClientException { + // Validate here too, not only at the controller: the timeseries_send_datapoint MCP tool calls + // this method directly and would otherwise bypass the batch and value-size constraints. + Set>> violations = validator.validate(data); + if (!violations.isEmpty()) { + throw new ConstraintViolationException(violations); + } + // Phase 1: resolve, authorise and validate. Needs the persistence context; no I/O. PreparedDatapointInsert prepared = transactionTemplate.execute(status -> prepareDatapointInsert(data)); @@ -830,6 +840,21 @@ private PreparedDatapointInsert prepareDatapointInsert(DataWrapper datapoints = entry.getDatapoints(); + if (datapoints == null || datapoints.size() <= FieldLimits.TEXT_DATAPOINTS_PER_COLLECTION_MAX) { + return; + } + ResponseError error = new ResponseError<>(); + BadRequestError badRequestError = new BadRequestError(); + badRequestError.setMessage(( + "A %s time series accepts at most %d data points per request, got %d. " + + "Split the batch into smaller requests." + ).formatted(ts.getValueType().getName(), FieldLimits.TEXT_DATAPOINTS_PER_COLLECTION_MAX, datapoints.size())); + error.setError(badRequestError); + throw new BadRequestException(error); + } + private static void addData( TimeseriesEntity ts, DataWrapperMessage dc, diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/websocket/DatapointListenWebSocketHandler.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/websocket/DatapointListenWebSocketHandler.java index 18f82e24..eec031e3 100644 --- a/datahub-api/src/main/java/ai/intellistream/datahub/api/websocket/DatapointListenWebSocketHandler.java +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/websocket/DatapointListenWebSocketHandler.java @@ -77,6 +77,14 @@ public class DatapointListenWebSocketHandler extends TextWebSocketHandler { private final JwtDecoder jwtDecoder; private final JsonMapper jsonMapper; private final StreamAccessAuthorizer accessAuthorizer; + private final WebSocketConnectionLimiter connectionLimiter; + /** Who each open session belongs to, so the ping can refresh it and close can free it. */ + private final Map owners = new ConcurrentHashMap<>(); + + /** The identity a socket is charged to. */ + private record ConnectionOwner(String tenantId, String subject) { + } + // Per-connection state — instance-local by design (a WebSocket is bound to one instance, so the // LB must route upgrades with session affinity). On instance failure the client reconnects. @@ -98,12 +106,14 @@ public DatapointListenWebSocketHandler(PulsarClient pulsarClient, TopicNames topicNames, JwtDecoder jwtDecoder, JsonMapper jsonMapper, - StreamAccessAuthorizer accessAuthorizer) { + StreamAccessAuthorizer accessAuthorizer, + WebSocketConnectionLimiter connectionLimiter) { this.pulsarClient = pulsarClient; this.topicNames = topicNames; this.jwtDecoder = jwtDecoder; this.jsonMapper = jsonMapper; this.accessAuthorizer = accessAuthorizer; + this.connectionLimiter = connectionLimiter; pingScheduler.scheduleAtFixedRate(this::sendPings, PING_INTERVAL_SECONDS, PING_INTERVAL_SECONDS, TimeUnit.SECONDS); } @@ -160,6 +170,19 @@ private void establishConnection(WebSocketSession rawSession) { return; } + // Concurrency cap, checked after authentication (so there is an identity to charge) and + // before the Pulsar consumer is built (so a refused connection costs the broker nothing). + var refusal = connectionLimiter.register(tenantId, jwt.getSubject(), rawSession.getId()); + if (refusal.isPresent()) { + log.info("Datapoint-listen handshake refused: {} limit of {} reached for tenant {}", + refusal.get().scope(), refusal.get().limit(), tenantId); + sendLimitError(rawSession, refusal.get()); + closeQuietly(rawSession, CloseStatus.POLICY_VIOLATION.withReason( + "WebSocket connection limit reached")); + return; + } + owners.put(rawSession.getId(), new ConnectionOwner(tenantId, jwt.getSubject())); + // Dataset ACL: narrow the requested interest set to timeseries whose dataset the caller may // read, so a caller can only live-tail data they are authorised for (matching the REST reads). DatasetPermissions permissions = accessAuthorizer.permissionsOf(tenantId, jwt); @@ -228,6 +251,10 @@ protected void handleTextMessage(WebSocketSession session, TextMessage message) @Override public void afterConnectionClosed(WebSocketSession session, CloseStatus status) { wsSessions.remove(session.getId()); + ConnectionOwner owner = owners.remove(session.getId()); + if (owner != null) { + connectionLimiter.release(owner.tenantId(), owner.subject(), session.getId()); + } DatapointListenSession listen = sessions.remove(session.getId()); if (listen != null) listen.stop(); log.info("Datapoint listen WS closed: session={} status={}", session.getId(), status); @@ -270,6 +297,24 @@ private Consumer buildConsumer() throws PulsarClientException { .subscribe(); } + /** + * One frame explaining the refusal, so a client sees a reason rather than a bare close code. + * Shaped like {@link SubscriptionWebSocketHandler}'s error frames, so a client that speaks to + * both sockets needs one parser rather than two. + */ + private void sendLimitError(WebSocketSession session, WebSocketConnectionLimiter.Refusal refusal) { + try { + session.sendMessage(new TextMessage(jsonMapper.writeValueAsString(Map.of( + "error", Boolean.TRUE, + "reason", "websocket-limit-reached", + "scope", refusal.scope(), + "limit", refusal.limit(), + "message", refusal.message())))); + } catch (Exception e) { + log.debug("Could not send the limit error frame to {}: {}", session.getId(), e.getMessage()); + } + } + private void sendPings() { if (wsSessions.isEmpty()) return; ByteBuffer empty = ByteBuffer.allocate(0); @@ -278,6 +323,12 @@ private void sendPings() { if (!session.isOpen()) continue; try { session.sendMessage(new PingMessage(empty)); + // Doubles as the liveness signal the connection registry counts by, so a socket + // this instance still holds is never mistaken for one left by a dead instance. + ConnectionOwner owner = owners.get(entry.getKey()); + if (owner != null) { + connectionLimiter.heartbeat(owner.tenantId(), owner.subject(), entry.getKey()); + } } catch (Exception e) { log.warn("Ping failed for datapoint-listen session {} ({}); leaving container to clean up", entry.getKey(), e.getMessage()); diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/websocket/SubscriptionListenSession.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/websocket/SubscriptionListenSession.java index 5bd47a2a..1cc8a05b 100644 --- a/datahub-api/src/main/java/ai/intellistream/datahub/api/websocket/SubscriptionListenSession.java +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/websocket/SubscriptionListenSession.java @@ -65,6 +65,11 @@ boolean hasStream(String externalId) { return streams.containsKey(externalId); } + /** How many subscriptions this one socket is currently multiplexing. */ + int streamCount() { + return streams.size(); + } + /** * Attach a subscription's consumer and start streaming it. Returns false (and does NOT take * ownership of the consumer) if the connection is stopping or the subscription is already diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/websocket/SubscriptionWebSocketHandler.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/websocket/SubscriptionWebSocketHandler.java index 201408ec..7dba69eb 100644 --- a/datahub-api/src/main/java/ai/intellistream/datahub/api/websocket/SubscriptionWebSocketHandler.java +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/websocket/SubscriptionWebSocketHandler.java @@ -98,6 +98,14 @@ public class SubscriptionWebSocketHandler extends TextWebSocketHandler { private final SubscriptionRepository subscriptionRepository; private final JsonMapper jsonMapper; private final StreamAccessAuthorizer accessAuthorizer; + private final WebSocketConnectionLimiter connectionLimiter; + /** Who each open session belongs to, so the ping can refresh it and close can free it. */ + private final Map owners = new ConcurrentHashMap<>(); + + /** The identity a socket is charged to. */ + private record ConnectionOwner(String tenantId, String subject) { + } + // Per-connection state — instance-local by design. A WebSocket is a TCP connection bound to one instance. No session affinity is required: each subscription's durable cursor lives in Pulsar, so a reconnect re-subscribes and resumes on whichever instance it lands on. private final Map sessions = new ConcurrentHashMap<>(); @@ -118,12 +126,14 @@ public SubscriptionWebSocketHandler(PulsarClient pulsarClient, TopicNames topicNames, SubscriptionRepository subscriptionRepository, JsonMapper jsonMapper, - StreamAccessAuthorizer accessAuthorizer) { + StreamAccessAuthorizer accessAuthorizer, + WebSocketConnectionLimiter connectionLimiter) { this.pulsarClient = pulsarClient; this.topicNames = topicNames; this.subscriptionRepository = subscriptionRepository; this.jsonMapper = jsonMapper; this.accessAuthorizer = accessAuthorizer; + this.connectionLimiter = connectionLimiter; pingScheduler.scheduleAtFixedRate(this::sendPings, PING_INTERVAL_SECONDS, PING_INTERVAL_SECONDS, TimeUnit.SECONDS); } @@ -142,6 +152,12 @@ private void sendPings() { if (!session.isOpen()) continue; try { session.sendMessage(new PingMessage(empty)); + // Doubles as the liveness signal the connection registry counts by, so a socket + // this instance still holds is never mistaken for one left by a dead instance. + ConnectionOwner owner = owners.get(entry.getKey()); + if (owner != null) { + connectionLimiter.heartbeat(owner.tenantId(), owner.subject(), entry.getKey()); + } } catch (Exception e) { log.warn("Ping failed for session {} ({}); leaving container to clean up", entry.getKey(), e.getMessage()); } @@ -156,6 +172,20 @@ public void afterConnectionEstablished(@NonNull WebSocketSession rawSession) thr return; } + // Concurrency cap, checked once the tenant is known and before any Pulsar consumer exists, + // so a refused connection costs the broker nothing. + String subject = extractSubject(rawSession.getPrincipal()); + var refusal = connectionLimiter.register(tenantId, subject, rawSession.getId()); + if (refusal.isPresent()) { + log.info("Subscription handshake refused: {} limit of {} reached for tenant {}", + refusal.get().scope(), refusal.get().limit(), tenantId); + sendLimitError(rawSession, refusal.get()); + closeQuietly(rawSession, CloseStatus.POLICY_VIOLATION.withReason( + "WebSocket connection limit reached")); + return; + } + owners.put(rawSession.getId(), new ConnectionOwner(tenantId, subject)); + List externalIds = parseExternalIds(rawSession); log.info("Subscription WS connect: tenant={} session={} initialSubscriptions={}", tenantId, rawSession.getId(), externalIds); @@ -236,6 +266,10 @@ protected void handleTextMessage(WebSocketSession session, TextMessage message) @Override public void afterConnectionClosed(WebSocketSession session, CloseStatus status) { wsSessions.remove(session.getId()); + ConnectionOwner owner = owners.remove(session.getId()); + if (owner != null) { + connectionLimiter.release(owner.tenantId(), owner.subject(), session.getId()); + } SubscriptionListenSession listen = sessions.remove(session.getId()); if (listen != null) listen.stop(); log.info("Subscription WS closed: session={} status={}", session.getId(), status); @@ -264,6 +298,17 @@ private void attachSubscription(SubscriptionListenSession listen, String tenantI String externalId, WebSocketSession session, DatasetPermissions permissions) { if (externalId == null || externalId.isBlank() || listen.hasStream(externalId)) return; + // One socket multiplexes many subscriptions, and each is a durable consumer the broker holds + // open. Counted in-session, which is exact — the socket lives on this instance. Over the cap + // is answered and the socket stays open: the other subscriptions on it are still valid. + int maxStreams = connectionLimiter.maxSubscriptionsPerSocket(); + if (maxStreams > 0 && listen.streamCount() >= maxStreams) { + log.info("Refusing subscription {} on session {}: at the {} per-socket limit", + externalId, session.getId(), maxStreams); + sendError(session, externalId, "subscription-limit-reached"); + return; + } + Optional maybe; try { maybe = loadSubscription(tenantId, externalId); @@ -376,6 +421,25 @@ private static List readStringArray(JsonNode node, String field) { } /** Notify the client that a requested subscription couldn't be attached. */ + /** One frame explaining the refusal, so a client sees a reason rather than a bare close code. */ + private void sendLimitError(WebSocketSession session, WebSocketConnectionLimiter.Refusal refusal) { + try { + session.sendMessage(new TextMessage(jsonMapper.writeValueAsString(Map.of( + "error", Boolean.TRUE, + "reason", "websocket-limit-reached", + "scope", refusal.scope(), + "limit", refusal.limit(), + "message", refusal.message())))); + } catch (Exception e) { + log.debug("Could not send the limit error frame to {}: {}", session.getId(), e.getMessage()); + } + } + + /** The JWT {@code sub} behind this connection, or null when the principal is not a JWT. */ + private static String extractSubject(Principal principal) { + return principal instanceof JwtAuthenticationToken jwtAuth ? jwtAuth.getToken().getSubject() : null; + } + private void sendError(WebSocketSession session, String externalId, String reason) { try { String json = jsonMapper.writeValueAsString(Map.of( diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/api/websocket/WebSocketConnectionLimiter.java b/datahub-api/src/main/java/ai/intellistream/datahub/api/websocket/WebSocketConnectionLimiter.java new file mode 100644 index 00000000..c16a6e90 --- /dev/null +++ b/datahub-api/src/main/java/ai/intellistream/datahub/api/websocket/WebSocketConnectionLimiter.java @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.websocket; + +import ai.intellistream.datahub.api.config.LimitsProperties; +import ai.intellistream.datahub.api.services.TenantLimits; +import ai.intellistream.datahub.api.services.TenantLimitsService; +import ai.intellistream.datahub.services.ValkeyService; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +/** + * Caps how many WebSocket connections one tenant, and one user inside it, may hold open at once. + * + *

A socket costs more than the request that opened it: a durable subscription keeps broker + * resources reserved whether or not anyone is reading, so an idle hoard is as expensive as a busy + * one, and datahub-cleanup only sweeps subscriptions nobody owns rather than ones somebody is + * holding on purpose. The per-minute rate limit does not reach this at all — a handshake happens + * once and the cost is what follows it. + * + *

Sockets live on whichever instance accepted them, so the count has to be shared. It is a Valkey + * sorted set per scope, scored by last heartbeat: connecting adds a member, the handlers' existing + * ping refreshes the score, closing removes it, and anything not refreshed inside + * {@link #STALE_AFTER_SECONDS} is ignored and swept. That last part is what makes an instance dying + * mid-connection self-correcting — its sockets age out instead of permanently occupying the budget. + * + *

A Valkey failure allows the connection. Refusing to open sockets because the registry is + * unreachable would turn a cache outage into an outage of the live features. + */ +@Slf4j +@Component +public class WebSocketConnectionLimiter { + + /** Comfortably beyond the handlers' 15s ping and the 45s idle timeout. */ + private static final long STALE_AFTER_SECONDS = 60; + + private final TenantLimitsService tenantLimits; + private final ValkeyService valkeyService; + private final LimitsProperties limits; + + public WebSocketConnectionLimiter(TenantLimitsService tenantLimits, + ValkeyService valkeyService, + LimitsProperties limits) { + this.tenantLimits = tenantLimits; + this.valkeyService = valkeyService; + this.limits = limits; + } + + /** Why a connection was refused, so the handler can tell the caller something useful. */ + public record Refusal(String scope, long limit) { + + public String message() { + return ("This %s already has %d open WebSocket connections, which is the limit. Close one, " + + "or contact IntelliStream to have the limit raised.").formatted(scope, limit); + } + } + + /** + * Register a new connection, or explain why it cannot be accepted. + * + * @return empty when the connection may proceed + */ + public Optional register(String tenantId, String subject, String sessionId) { + if (tenantId == null || sessionId == null) { + return Optional.empty(); + } + TenantLimits effective = tenantLimits.forTenant(tenantId); + if (effective == null) { + return Optional.empty(); + } + + try { + long tenantLimit = effective.maxWsSocketsPerTenant(); + if (!TenantLimits.unlimited(tenantLimit) + && valkeyService.countLiveMembers(tenantKey(tenantId), STALE_AFTER_SECONDS) >= tenantLimit) { + return Optional.of(new Refusal("tenant", tenantLimit)); + } + + long userLimit = effective.maxWsSocketsPerUser(); + if (subject != null && !TenantLimits.unlimited(userLimit) + && valkeyService.countLiveMembers(userKey(tenantId, subject), STALE_AFTER_SECONDS) >= userLimit) { + return Optional.of(new Refusal("user", userLimit)); + } + + heartbeat(tenantId, subject, sessionId); + } catch (RuntimeException e) { + log.warn("WebSocket connection limiting unavailable ({}); allowing the connection.", e.toString()); + } + return Optional.empty(); + } + + /** Refresh this connection's score, so it keeps counting. Called from the handlers' ping. */ + public void heartbeat(String tenantId, String subject, String sessionId) { + if (tenantId == null || sessionId == null) { + return; + } + try { + valkeyService.touchMember(tenantKey(tenantId), sessionId, STALE_AFTER_SECONDS); + if (subject != null) { + valkeyService.touchMember(userKey(tenantId, subject), sessionId, STALE_AFTER_SECONDS); + } + } catch (RuntimeException e) { + log.debug("Could not refresh WebSocket registration for {}: {}", sessionId, e.toString()); + } + } + + /** Free the slot. Best-effort: a missed release ages out on its own. */ + public void release(String tenantId, String subject, String sessionId) { + if (tenantId == null || sessionId == null) { + return; + } + try { + valkeyService.removeMember(tenantKey(tenantId), sessionId); + if (subject != null) { + valkeyService.removeMember(userKey(tenantId, subject), sessionId); + } + } catch (RuntimeException e) { + log.debug("Could not release WebSocket registration for {}: {}", sessionId, e.toString()); + } + } + + /** + * The per-socket subscription cap. Deployment-wide rather than per tenant: it bounds one + * socket's fan-out, and an in-session counter is exact, so it needs no shared state. + */ + public int maxSubscriptionsPerSocket() { + return limits.getWebsocket().getMaxSubscriptionsPerSocket(); + } + + private static String tenantKey(String tenantId) { + return "dh:ws:t:" + tenantId; + } + + private static String userKey(String tenantId, String subject) { + return "dh:ws:u:%s:%s".formatted(tenantId, subject); + } +} diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/config/FilterConfig.java b/datahub-api/src/main/java/ai/intellistream/datahub/config/FilterConfig.java index 0348de0a..91712dad 100644 --- a/datahub-api/src/main/java/ai/intellistream/datahub/config/FilterConfig.java +++ b/datahub-api/src/main/java/ai/intellistream/datahub/config/FilterConfig.java @@ -1,9 +1,12 @@ // SPDX-License-Identifier: AGPL-3.0-or-later package ai.intellistream.datahub.config; +import ai.intellistream.datahub.api.config.LimitsProperties; import ai.intellistream.datahub.api.filters.CachingBodyFilter; +import ai.intellistream.datahub.api.filters.RequestBodySizeLimitFilter; import ai.intellistream.datahub.api.filters.RequestLogFilter; import ai.intellistream.datahub.api.filters.RequestStateCleanupFilter; +import ai.intellistream.datahub.api.services.IngestQuotaService; import jakarta.servlet.Filter; import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.context.annotation.Bean; @@ -33,6 +36,19 @@ public FilterRegistrationBean logFilter() { return rlf; } + @Bean + public FilterRegistrationBean requestBodySizeLimitFilter(LimitsProperties limits, + IngestQuotaService ingestQuota) { + final FilterRegistrationBean f = + new FilterRegistrationBean<>(new RequestBodySizeLimitFilter(limits, ingestQuota)); + f.addUrlPatterns("/*"); + // Before the body cache (0), so an oversized body is refused rather than buffered; after the + // security chain (-100), so an unauthenticated caller still gets 401 rather than a hint about + // what the limits are. + f.setOrder(-1); + return f; + } + @Bean public FilterRegistrationBean bodyCacheFilter(){ final FilterRegistrationBean cbf = new FilterRegistrationBean<>(new CachingBodyFilter()); diff --git a/datahub-api/src/main/java/ai/intellistream/datahub/config/SecurityConfig.java b/datahub-api/src/main/java/ai/intellistream/datahub/config/SecurityConfig.java index f40546c2..3a8e54e9 100644 --- a/datahub-api/src/main/java/ai/intellistream/datahub/config/SecurityConfig.java +++ b/datahub-api/src/main/java/ai/intellistream/datahub/config/SecurityConfig.java @@ -1,7 +1,11 @@ // SPDX-License-Identifier: AGPL-3.0-or-later package ai.intellistream.datahub.config; +import ai.intellistream.datahub.api.config.LimitsProperties; +import ai.intellistream.datahub.api.filters.RateLimitFilter; import ai.intellistream.datahub.api.filters.TenantProvisioningFilter; +import ai.intellistream.datahub.api.services.TenantLimitsService; +import ai.intellistream.datahub.services.ValkeyService; import ai.intellistream.datahub.tenant.TenantConfigService; import ai.intellistream.datahub.tenant.TenantContext; import lombok.extern.slf4j.Slf4j; @@ -70,6 +74,9 @@ SecurityFilterChain filterChain( ServerProperties serverProperties, ObjectProvider tenantConfigService, ObjectProvider tenantMigrator, + LimitsProperties limitsProperties, + ObjectProvider tenantLimitsService, + ObjectProvider valkeyService, @Value("${origins:http://localhost:8080}") String[] origins, @Value("${permit-all:[]}") String[] permitAll ) { @@ -109,6 +116,19 @@ SecurityFilterChain filterChain( ) ); + // Rate limiting runs on the authenticated identity, so it goes after authz — and is added + // before the provisioning filter below so it sits ahead of it in the chain, turning an + // over-budget caller away before the request costs a Vault lookup or a Flyway check. + // /mcp/* rides this same chain, which is how the MCP tools end up on the REST budget. + // Constructed rather than injected as a bean: a Filter bean would also be picked up by + // Boot's servlet auto-registration and run a second time outside this chain. + TenantLimitsService limitsService = tenantLimitsService.getIfAvailable(); + ValkeyService valkey = valkeyService.getIfAvailable(); + if (limitsService != null && valkey != null) { + http.addFilterAfter(new RateLimitFilter(limitsProperties, limitsService, valkey), + AuthorizationFilter.class); + } + // Refuse unknown tenants (403) and provision the request's tenant schema on first touch, // AFTER authz has run and OrganizationValidator has set TenantContext from the JWT. Gated // on TenantConfigService rather than on the migrator, so the unknown-tenant check still diff --git a/datahub-api/src/main/resources/application.yml b/datahub-api/src/main/resources/application.yml index 5e359153..80be0382 100644 --- a/datahub-api/src/main/resources/application.yml +++ b/datahub-api/src/main/resources/application.yml @@ -135,6 +135,74 @@ datahub: # an explicit `policy`/`streaming` value in Vault overrides these per tenant. policy: true streaming: true + limits: + # Largest accepted request body, in bytes. This is the application's own ceiling, independent of + # nginx: anything that can reach the service port would otherwise be unbounded. Over the limit is + # a 413, which the SDK treats as terminal rather than retrying. + # Kept under Pulsar's 5 MB per-message default — an event create batch is published as one + # message, so this is what keeps that message legal. + # 0 or negative disables the check. + max-body-bytes: 4194304 + # POST /timeseries/data only. Higher because a full 100 000-point numeric batch is around 5 MB of + # JSON by itself, and a maxed-out text batch around 11 MB. + max-body-bytes-datapoints: 16777216 + # How long a tenant's tenant_limits row is cached in-process. Also the propagation delay: an + # UPDATE to that table takes effect across every instance within this window, no restart needed. + cache-ttl: 5m + rate: + # Requests per minute, counted in a fixed window per tenant and per user. Over the budget is a + # 429 with Retry-After; the SDK already treats that as retryable and backs off. + # A tenant's tenant_limits row overrides any of these. 0 or negative disables one. + # If Valkey is unreachable the limiter allows traffic through rather than refusing it. + enabled: true + # The tenant is the real budget — a public signup gets an organization of its own. The + # per-user figures stop a single identity inside a tenant spending all of it. + write-per-minute-per-tenant: 2000 + read-per-minute-per-tenant: 6000 + write-per-minute-per-user: 600 + read-per-minute-per-user: 1200 + quota: + # Daily ingest allowance per tenant, reset at 00:00 UTC. Over it is a 429 with Retry-After + # pointing at midnight, which the SDK spools and replays. Counted in memory and flushed to + # Valkey every couple of seconds, so a tenant can overshoot slightly across instances — these + # are ceilings, not invoices. + enabled: true + events-per-day: 100000 + # Resources, time series, data sets, labels, policies and functions are all rows in the node + # table, so they share one allowance. + nodes-per-day: 50000 + edges-per-day: 100000 + datapoints-per-day: 10000000 + # Bytes of write-request body. The one quota that really bounds storage growth: an entity + # count does not, because a single legal entity can be a few hundred KB. + ingest-bytes-per-day: 1073741824 + lifetime: + # How large a tenant may grow, rather than how fast. Over one of these is a 403 saying how it + # is raised: waiting will not clear it, so it is not a 429. + # + # OFF by default, unlike the rate limits and daily quotas, because these numbers size a free + # playground rather than a real deployment. Switching this on applies them to every tenant + # that has no tenant_limits override, so a tenant already holding more than max-resources + # would start refusing writes the moment the api restarted. Give the tenants that should keep + # growing a 0 in their tenant_limits row FIRST, then enable this. + enabled: false + # Counted live against the node table, so deleting frees room again. + max-resources: 1000 + max-events-total: 25000 + max-datapoints-total: 1000000000 + # Text and mixed series are the expensive kind to store and query, so they are capped well + # below the numeric total. + max-text-datapoints-total: 100000 + websocket: + # Concurrent sockets, and subscriptions multiplexed over one of them. Capped separately from + # requests because a socket's cost is what follows the handshake: a durable subscription holds + # broker resources whether or not anyone reads it, so an idle hoard costs as much as a busy one + # and the per-minute rate limit never sees it. Over the cap gets an error frame naming the + # limit and a 1008 close. The socket counts are per-tenant overridable via tenant_limits. + enabled: true + max-sockets-per-tenant: 10 + max-sockets-per-user: 10 + max-subscriptions-per-socket: 10 files: checksum: # Checksum algorithm applied to every uploaded file. diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/controllers/errors/LimitExceptionHandlerTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/controllers/errors/LimitExceptionHandlerTest.java new file mode 100644 index 00000000..21098cb6 --- /dev/null +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/controllers/errors/LimitExceptionHandlerTest.java @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.controllers.errors; + +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ProblemDetail; +import org.springframework.http.ResponseEntity; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The problem body's detail is the sentence the limit exception composes from its own numbers, + * reached through {@link LimitException#detail()} rather than the exception message, so nothing + * that arrived from a request or a lower layer can end up in a response. + */ +class LimitExceptionHandlerTest { + + private final LimitExceptionHandler handler = new LimitExceptionHandler(); + + @Test + void quotaRefusalCarriesTheComposedDetailAndTheRetryAfter() { + var refusal = new IngestQuotaExceededException("events", 100_000, 43_200); + + ResponseEntity response = handler.handleQuotaExceeded(refusal); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.TOO_MANY_REQUESTS); + assertThat(response.getHeaders().getFirst(HttpHeaders.RETRY_AFTER)).isEqualTo("43200"); + ProblemDetail body = response.getBody(); + assertThat(body).isNotNull(); + assertThat(body.getDetail()).isEqualTo(refusal.detail()) + .isEqualTo("Daily events ingest quota (100000) is spent; it resets at 00:00 UTC."); + assertThat(body.getProperties()).containsEntry("metric", "events").containsEntry("limit", 100_000L); + } + + @Test + void ceilingRefusalCarriesTheComposedDetailAndNoRetryAfter() { + var refusal = new TenantLimitReachedException("events", 25_000); + + ProblemDetail body = handler.handleTenantLimitReached(refusal); + + assertThat(body.getStatus()).isEqualTo(HttpStatus.FORBIDDEN.value()); + assertThat(body.getDetail()).isEqualTo(refusal.detail()).contains("25000 events"); + assertThat(body.getProperties()).containsEntry("metric", "events").containsEntry("limit", 25_000L); + } +} diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/filters/RateLimitFilterTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/filters/RateLimitFilterTest.java new file mode 100644 index 00000000..25dbad8e --- /dev/null +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/filters/RateLimitFilterTest.java @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.filters; + +import ai.intellistream.datahub.api.config.LimitsProperties; +import ai.intellistream.datahub.api.services.TenantLimits; +import ai.intellistream.datahub.api.services.TenantLimitsService; +import ai.intellistream.datahub.services.ValkeyService; +import ai.intellistream.datahub.tenant.TenantContext; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.mock.web.MockFilterChain; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.context.SecurityContextHolder; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The per-minute budget, and the two ways it must not misfire: it has to charge the right identity, + * and it has to let traffic through when it cannot count at all. + */ +class RateLimitFilterTest { + + private static final String TENANT = "2c5e2e73-2c2e-4516-ab58-4e602e1c495b"; + + private final LimitsProperties limits = new LimitsProperties(); + private final TenantLimitsService tenantLimits = mock(TenantLimitsService.class); + private final ValkeyService valkeyService = mock(ValkeyService.class); + private final RateLimitFilter filter = new RateLimitFilter(limits, tenantLimits, valkeyService); + + private final MockHttpServletResponse response = new MockHttpServletResponse(); + private final MockFilterChain chain = new MockFilterChain(); + + /** Rate limits only; the quota and lifetime figures are unlimited here. */ + private static TenantLimits limitsOf(int writeTenant, int readTenant, int writeUser, int readUser) { + return new TenantLimits(writeTenant, readTenant, writeUser, readUser, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + } + + @BeforeEach + void setUp() { + TenantContext.setTenantId(TENANT); + when(tenantLimits.forTenant(anyString())).thenReturn(limitsOf(10, 20, 5, 8)); + } + + @AfterEach + void tearDown() { + TenantContext.clear(); + SecurityContextHolder.clearContext(); + } + + private void authenticateAs(String subject) { + SecurityContextHolder.getContext().setAuthentication( + new UsernamePasswordAuthenticationToken(subject, "n/a", List.of())); + } + + /** Every counter reports {@code count}, so one stub drives whichever key is checked first. */ + private void countsReach(long count) { + when(valkeyService.incrementAndExpireIfNew(anyString(), anyLong(), anyLong())).thenReturn(count); + } + + @Test + void underTheBudgetPassesThrough() throws Exception { + countsReach(3); + + filter.doFilter(new MockHttpServletRequest("POST", "/events/create"), response, chain); + + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + assertThat(chain.getRequest()).isNotNull(); + } + + @Test + void overTheTenantBudgetIsRefusedWithRetryAfter() throws Exception { + countsReach(11); // the tenant write budget is 10 + + filter.doFilter(new MockHttpServletRequest("POST", "/events/create"), response, chain); + + assertThat(response.getStatus()).isEqualTo(HttpStatus.TOO_MANY_REQUESTS.value()); + assertThat(response.getContentType()).startsWith(MediaType.APPLICATION_PROBLEM_JSON_VALUE); + assertThat(response.getContentAsString()).contains("rate-limit-exceeded", "\"scope\":\"tenant\""); + assertThat(chain.getRequest()).as("the chain must not run").isNull(); + + int retryAfter = Integer.parseInt(response.getHeader(HttpHeaders.RETRY_AFTER)); + assertThat(retryAfter).isBetween(1, 60); + } + + @Test + void readsAndWritesHaveSeparateBudgets() throws Exception { + // 11 is over the write budget (10) but well under the read budget (20), so a GET survives + // a count that would refuse a POST. + countsReach(11); + + filter.doFilter(new MockHttpServletRequest("GET", "/events"), response, chain); + + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + } + + @Test + void aPostThatOnlyReadsIsChargedToTheReadBudget() throws Exception { + // /events/filter, /resources/search and friends POST only because they carry a filter body. + // Charging them to the write budget would throttle browsing long before anyone wrote data. + countsReach(11); // over the write budget (10), under the read one (20) + + filter.doFilter(new MockHttpServletRequest("POST", "/events/filter"), response, chain); + + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + verify(valkeyService).incrementAndExpireIfNew( + org.mockito.ArgumentMatchers.matches("dh:rl:t:" + TENANT + ":r:\\d+"), anyLong(), anyLong()); + } + + @Test + void aRealWriteIsStillChargedToTheWriteBudget() throws Exception { + countsReach(11); + + filter.doFilter(new MockHttpServletRequest("POST", "/events/create"), response, chain); + + assertThat(response.getStatus()).isEqualTo(HttpStatus.TOO_MANY_REQUESTS.value()); + } + + @Test + void aDeleteIsAWriteWhateverItsPathLooksLike() throws Exception { + countsReach(11); + + filter.doFilter(new MockHttpServletRequest("DELETE", "/events/delete"), response, chain); + + assertThat(response.getStatus()).isEqualTo(HttpStatus.TOO_MANY_REQUESTS.value()); + } + + @Test + void theUserBudgetAppliesInsideTheTenantBudget() throws Exception { + authenticateAs("user-1"); + countsReach(6); // under the tenant write budget (10), over the user one (5) + + filter.doFilter(new MockHttpServletRequest("POST", "/events/create"), response, chain); + + assertThat(response.getStatus()).isEqualTo(HttpStatus.TOO_MANY_REQUESTS.value()); + assertThat(response.getContentAsString()).contains("\"scope\":\"user\""); + } + + @Test + void anUnlimitedBudgetIsNotEvenCounted() throws Exception { + when(tenantLimits.forTenant(anyString())).thenReturn(limitsOf(0, 0, 0, 0)); + + filter.doFilter(new MockHttpServletRequest("POST", "/events/create"), response, chain); + + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + verify(valkeyService, never()).incrementAndExpireIfNew(anyString(), anyLong(), anyLong()); + } + + @Test + void requestsWithNoTenantAreNotCharged() throws Exception { + // A permit-all endpoint: swagger, the session routes, the live-tail handshake. + TenantContext.clear(); + + filter.doFilter(new MockHttpServletRequest("GET", "/swagger-ui/index.html"), response, chain); + + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + verify(valkeyService, never()).incrementAndExpireIfNew(anyString(), anyLong(), anyLong()); + } + + @Test + void aValkeyOutageLetsTrafficThrough() throws Exception { + // Refusing everything because the counter is unavailable would turn a cache outage into a + // full outage. The size and batch caps still apply underneath. + when(valkeyService.incrementAndExpireIfNew(anyString(), anyLong(), anyLong())) + .thenThrow(new IllegalStateException("valkey down")); + + filter.doFilter(new MockHttpServletRequest("POST", "/events/create"), response, chain); + + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + assertThat(chain.getRequest()).isNotNull(); + } + + @Test + void disablingTheLimiterSkipsItEntirely() throws Exception { + limits.getRate().setEnabled(false); + + filter.doFilter(new MockHttpServletRequest("POST", "/events/create"), response, chain); + + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + verify(valkeyService, never()).incrementAndExpireIfNew(anyString(), anyLong(), anyLong()); + } + + @Test + void theWindowKeyCarriesTenantMethodClassAndMinute() throws Exception { + countsReach(1); + + filter.doFilter(new MockHttpServletRequest("POST", "/events/create"), response, chain); + + // Tenant-scoped, write-scoped, and per-minute: without the minute the window never rolls, + // and without the tenant two customers would share one budget. + verify(valkeyService).incrementAndExpireIfNew( + org.mockito.ArgumentMatchers.matches("dh:rl:t:" + TENANT + ":w:\\d+"), anyLong(), anyLong()); + } +} diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/filters/RequestBodySizeLimitFilterTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/filters/RequestBodySizeLimitFilterTest.java new file mode 100644 index 00000000..17cca89e --- /dev/null +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/filters/RequestBodySizeLimitFilterTest.java @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.filters; + +import ai.intellistream.datahub.api.config.LimitsProperties; +import ai.intellistream.datahub.api.services.IngestQuotaService; +import jakarta.servlet.ServletException; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.mock.web.MockFilterChain; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * The application's own body ceiling. nginx caps bodies too, but the api is reachable directly on + * its service port, so the limit has to exist here or it does not exist at all. + */ +class RequestBodySizeLimitFilterTest { + + private final LimitsProperties limits = new LimitsProperties(); + private final RequestBodySizeLimitFilter filter = new RequestBodySizeLimitFilter(limits, mock(IngestQuotaService.class)); + private final MockHttpServletResponse response = new MockHttpServletResponse(); + private final MockFilterChain chain = new MockFilterChain(); + + private static MockHttpServletRequest post(String uri, int bodyBytes) { + MockHttpServletRequest request = new MockHttpServletRequest("POST", uri); + request.setContent("x".repeat(bodyBytes).getBytes(StandardCharsets.UTF_8)); + request.setContentType(MediaType.APPLICATION_JSON_VALUE); + return request; + } + + @Test + void bodyUnderTheLimitPassesThrough() throws Exception { + limits.setMaxBodyBytes(1024); + filter.doFilter(post("/events/create", 512), response, chain); + + assertThat(chain.getRequest()).isNotNull(); + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + } + + @Test + void declaredContentLengthOverTheLimitIsRejectedBeforeTheBodyIsRead() throws Exception { + limits.setMaxBodyBytes(1024); + filter.doFilter(post("/events/create", 2048), response, chain); + + assertThat(response.getStatus()).isEqualTo(HttpStatus.PAYLOAD_TOO_LARGE.value()); + assertThat(response.getContentType()).startsWith(MediaType.APPLICATION_PROBLEM_JSON_VALUE); + assertThat(response.getContentAsString()).contains("request-too-large", "\"limitBytes\":1024"); + assertThat(chain.getRequest()).as("the chain must not run").isNull(); + } + + /** A chunked request: bytes arrive, but no {@code Content-Length} declares how many. */ + private static MockHttpServletRequest chunkedPost(String uri, int bodyBytes) { + MockHttpServletRequest request = new MockHttpServletRequest("POST", uri) { + @Override + public long getContentLengthLong() { + return -1; + } + + @Override + public int getContentLength() { + return -1; + } + }; + request.setContent("x".repeat(bodyBytes).getBytes(StandardCharsets.UTF_8)); + request.setContentType(MediaType.APPLICATION_JSON_VALUE); + request.addHeader("Transfer-Encoding", "chunked"); + return request; + } + + /** A chain that actually consumes the body, which is what drives the counting stream. */ + private static MockFilterChain readingChain() { + return new MockFilterChain() { + @Override + public void doFilter(jakarta.servlet.ServletRequest req, jakarta.servlet.ServletResponse res) + throws IOException, ServletException { + req.getInputStream().readAllBytes(); + super.doFilter(req, res); + } + }; + } + + @Test + void undeclaredLengthUnderTheLimitReadsThrough() throws Exception { + limits.setMaxBodyBytes(4096); + MockFilterChain chain = readingChain(); + + filter.doFilter(chunkedPost("/events/create", 512), response, chain); + + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + assertThat(chain.getRequest()).isNotNull(); + } + + @Test + void undeclaredLengthOverTheLimitFailsMidRead() { + // Nothing declared the size, so the cap can only be applied while the bytes are consumed. + limits.setMaxBodyBytes(64); + + assertThatThrownBy(() -> filter.doFilter(chunkedPost("/events/create", 4096), response, readingChain())) + .isInstanceOf(RequestBodySizeLimitFilter.RequestBodyTooLargeException.class) + .hasMessageContaining("64"); + } + + @Test + void datapointInsertsGetTheirOwnLargerCeiling() throws Exception { + limits.setMaxBodyBytes(1024); + limits.setMaxBodyBytesDatapoints(8192); + + filter.doFilter(post("/timeseries/data", 4096), response, chain); + + assertThat(response.getStatus()) + .as("4 KB is over the general cap but under the datapoint cap") + .isEqualTo(HttpStatus.OK.value()); + assertThat(chain.getRequest()).isNotNull(); + } + + @Test + void fileUploadIsExemptSoLargeUploadsStillStream() throws Exception { + limits.setMaxBodyBytes(64); + MockHttpServletRequest upload = new MockHttpServletRequest("PUT", "/files"); + upload.setContent("x".repeat(4096).getBytes(StandardCharsets.UTF_8)); + + filter.doFilter(upload, response, chain); + + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + assertThat(chain.getRequest()).isNotNull(); + } + + @Test + void fileDownloadIsExempt() throws Exception { + limits.setMaxBodyBytes(64); + MockHttpServletRequest download = new MockHttpServletRequest("GET", "/files/download/abc"); + download.setContent("x".repeat(4096).getBytes(StandardCharsets.UTF_8)); + + filter.doFilter(download, response, chain); + + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + assertThat(chain.getRequest()).isNotNull(); + } + + @Test + void aZeroLimitDisablesTheCheck() throws Exception { + limits.setMaxBodyBytes(0); + filter.doFilter(post("/events/create", 8192), response, chain); + + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + assertThat(chain.getRequest()).isNotNull(); + } +} diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/EventServiceTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/EventServiceTest.java index b6980ac5..ae046181 100644 --- a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/EventServiceTest.java +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/EventServiceTest.java @@ -52,6 +52,8 @@ class EventServiceTest { @Mock private EventDimensionRepository eventDimensionRepository; @Mock private DataSetRepository dataSetRepository; @Mock private ai.intellistream.datahub.api.datasecurity.DatasetClosureService datasetClosureService; + // Charged on create, so it has to exist even where the test is not about quotas. + @Mock private IngestQuotaService ingestQuota; /** * By default a data set has no children, so the closure is the roots themselves — that keeps diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/IngestQuotaServiceTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/IngestQuotaServiceTest.java new file mode 100644 index 00000000..81e18064 --- /dev/null +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/IngestQuotaServiceTest.java @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.services; + +import ai.intellistream.datahub.api.controllers.errors.IngestQuotaExceededException; +import ai.intellistream.datahub.api.controllers.errors.TenantLimitReachedException; +import ai.intellistream.datahub.api.services.IngestQuotaService.QuotaMetric; +import ai.intellistream.datahub.services.ValkeyService; +import ai.intellistream.datahub.tenant.TenantContext; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The two ceilings and how they differ: a daily allowance that comes back, and a lifetime one that + * does not. Both are counted locally and flushed, so the checks have to see uncommitted local + * spending as well as the flushed total. + */ +class IngestQuotaServiceTest { + + private static final String TENANT = "acme"; + + private final TenantLimitsService tenantLimits = mock(TenantLimitsService.class); + private final ValkeyService valkeyService = mock(ValkeyService.class); + private final AtomicReference now = + new AtomicReference<>(Instant.parse("2026-08-25T12:00:00Z")); + + private IngestQuotaService service; + + private static TenantLimits limits(long eventsPerDay, long maxEventsTotal) { + return new TenantLimits(0, 0, 0, 0, + eventsPerDay, 0, 0, 0, 0, + 0, maxEventsTotal, 0, 0, + 0, 0); + } + + @BeforeEach + void setUp() { + TenantContext.setTenantId(TENANT); + service = new IngestQuotaService(tenantLimits, valkeyService, now::get); + when(tenantLimits.forTenant(anyString())).thenReturn(limits(100, 0)); + } + + @AfterEach + void tearDown() { + TenantContext.clear(); + } + + @Test + void underTheDailyAllowanceIsAccepted() { + assertThatCode(() -> service.checkAndRecord(QuotaMetric.EVENTS, 50)).doesNotThrowAnyException(); + } + + @Test + void spendingIsCumulativeWithinTheWindow() { + // Both calls are under the limit alone; together they are over it. Nothing has flushed yet, + // so this only passes if a check counts what this instance is still holding. + service.checkAndRecord(QuotaMetric.EVENTS, 60); + + assertThatThrownBy(() -> service.checkAndRecord(QuotaMetric.EVENTS, 60)) + .isInstanceOf(IngestQuotaExceededException.class); + } + + @Test + void aRefusedBatchIsNotCharged() { + service.checkAndRecord(QuotaMetric.EVENTS, 60); + + assertThatThrownBy(() -> service.checkAndRecord(QuotaMetric.EVENTS, 60)) + .isInstanceOf(IngestQuotaExceededException.class); + + // The refused 60 must not have been added, or the tenant would be punished twice for it. + assertThatCode(() -> service.checkAndRecord(QuotaMetric.EVENTS, 40)).doesNotThrowAnyException(); + } + + @Test + void theQuotaRefusalPointsAtTheNextUtcMidnight() { + service.checkAndRecord(QuotaMetric.EVENTS, 100); + + assertThatThrownBy(() -> service.checkAndRecord(QuotaMetric.EVENTS, 1)) + .isInstanceOfSatisfying(IngestQuotaExceededException.class, e -> { + // 12:00Z, so twelve hours of it. + assertThat(e.getRetryAfterSeconds()).isEqualTo(12 * 3600); + assertThat(e.getLimit()).isEqualTo(100); + }); + } + + @Test + void aNewDayIsANewAllowance() { + service.checkAndRecord(QuotaMetric.EVENTS, 100); + assertThatThrownBy(() -> service.checkAndRecord(QuotaMetric.EVENTS, 1)) + .isInstanceOf(IngestQuotaExceededException.class); + + now.set(Instant.parse("2026-08-26T00:30:00Z")); + + // A different day means a different key, so the tenant starts over rather than staying + // refused until someone intervenes. + assertThatCode(() -> service.checkAndRecord(QuotaMetric.EVENTS, 100)).doesNotThrowAnyException(); + } + + @Test + void theLifetimeCeilingIsSeparateAndNotRetryable() { + when(tenantLimits.forTenant(anyString())).thenReturn(limits(1_000_000, 25)); + + assertThatThrownBy(() -> service.checkAndRecord(QuotaMetric.EVENTS, 26)) + .isInstanceOf(TenantLimitReachedException.class) + .hasMessageContaining("Contact IntelliStream"); + } + + @Test + void aNewDayDoesNotResetTheLifetimeCeiling() { + when(tenantLimits.forTenant(anyString())).thenReturn(limits(1_000_000, 25)); + service.checkAndRecord(QuotaMetric.EVENTS, 25); + + now.set(Instant.parse("2026-09-01T00:00:00Z")); + + // The whole point of a lifetime ceiling: waiting does not clear it. + assertThatThrownBy(() -> service.checkAndRecord(QuotaMetric.EVENTS, 1)) + .isInstanceOf(TenantLimitReachedException.class); + } + + @Test + void anUnlimitedMetricIsNotCounted() { + when(tenantLimits.forTenant(anyString())).thenReturn(limits(0, 0)); + + service.checkAndRecord(QuotaMetric.EVENTS, 1_000_000); + service.flush(); + + verify(valkeyService, never()).incrementAndExpireIfNew(anyString(), anyLong(), anyLong()); + } + + @Test + void flushPushesTheDailyCounterWithAnExpiryAndTheLifetimeOneWithout() { + when(tenantLimits.forTenant(anyString())).thenReturn(limits(1000, 5000)); + service.checkAndRecord(QuotaMetric.EVENTS, 7); + + service.flush(); + + // A daily key must expire or the window never rolls; a lifetime total must not. + verify(valkeyService).incrementAndExpireIfNew(contains(":events:" + TENANT + ":2026"), anyLong(), anyLong()); + verify(valkeyService).increment(contains("dh:quota:total:events:" + TENANT), anyLong()); + } + + @Test + void aFailedFlushKeepsTheDeltaForTheNextAttempt() { + when(valkeyService.incrementAndExpireIfNew(anyString(), anyLong(), anyLong())) + .thenThrow(new IllegalStateException("valkey down")); + service.checkAndRecord(QuotaMetric.EVENTS, 9); + + service.flush(); + service.flush(); + + // Two attempts for one delta: losing it would let a tenant re-spend what it already used. + verify(valkeyService, atLeastOnce()).incrementAndExpireIfNew(anyString(), anyLong(), anyLong()); + } + + @Test + void anUnreadableCounterDoesNotRefuseIngest() { + when(valkeyService.getString(anyString())).thenThrow(new IllegalStateException("valkey down")); + + assertThatCode(() -> service.checkAndRecord(QuotaMetric.EVENTS, 1)).doesNotThrowAnyException(); + } + + @Test + void requestsWithNoTenantAreNotCharged() { + TenantContext.clear(); + + service.checkAndRecord(QuotaMetric.EVENTS, 1_000_000); + service.flush(); + + verify(valkeyService, never()).incrementAndExpireIfNew(anyString(), anyLong(), anyLong()); + } +} diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceFilterDatasetExpansionTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceFilterDatasetExpansionTest.java index 92560d40..e229521a 100644 --- a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceFilterDatasetExpansionTest.java +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceFilterDatasetExpansionTest.java @@ -56,6 +56,7 @@ entityManager, mock(NodeRepository.class), mock(NodeService.class), mock(EdgeRep mock(DataSecurity.class), mock(SubscriptionRepository.class), mock(Validator.class), mock(PolicyEnforcement.class), closure, + mock(IngestQuotaService.class), mock(TenantLimitsService.class), new ai.intellistream.datahub.api.edge.EdgeMapper(mock(NodeRepository.class), mock(RelationshipTypeRepository.class), mock(RelationshipTypeService.class)), new ai.intellistream.datahub.api.services.node.NodeUpdateService( mock(NodeRepository.class), mock(DataSetRepository.class), mock(DataSecurity.class), diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceAclInvalidationTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceAclInvalidationTest.java index b024e02c..81512852 100644 --- a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceAclInvalidationTest.java +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceAclInvalidationTest.java @@ -93,7 +93,8 @@ class ResourceServiceAclInvalidationTest { relationshipTypeRepository, relationshipTypeService, eventPublisher, graphOutbox, neo4JService, dataSecurity, subscriptionRepository, validator, policyEnforcement, datasetClosureService, - new ai.intellistream.datahub.api.edge.EdgeMapper(nodeRepository, relationshipTypeRepository, relationshipTypeService), + mock(IngestQuotaService.class), mock(TenantLimitsService.class), + new ai.intellistream.datahub.api.edge.EdgeMapper(nodeRepository, relationshipTypeRepository, relationshipTypeService), new ai.intellistream.datahub.api.services.node.NodeUpdateService( nodeRepository, dataSetRepository, dataSecurity, labelService, nodeService, policyEnforcement), mock(ai.intellistream.datahub.api.policy.NamingPolicyResolver.class)); diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceCreateEventTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceCreateEventTest.java index 5497ee8d..d676dea5 100644 --- a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceCreateEventTest.java +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceCreateEventTest.java @@ -75,6 +75,7 @@ class ResourceServiceCreateEventTest { entityManager, nodeRepository, nodeService, edgeRepository, relationshipTypeRepository, relationshipTypeService, eventPublisher, graphOutbox, neo4JService, dataSecurity, subscriptionRepository, validator, policyEnforcement, datasetClosureService, + mock(IngestQuotaService.class), mock(TenantLimitsService.class), new EdgeMapper(nodeRepository, relationshipTypeRepository, relationshipTypeService), new ai.intellistream.datahub.api.services.node.NodeUpdateService( nodeRepository, dataSetRepository, dataSecurity, labelService, nodeService, policyEnforcement), diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceCreateGuardsTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceCreateGuardsTest.java index 41db03c2..3527d0b0 100644 --- a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceCreateGuardsTest.java +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceCreateGuardsTest.java @@ -71,6 +71,7 @@ relationshipTypeRepository, mock(RelationshipTypeService.class), mock(ApplicationEventPublisher.class), mock(GraphOutbox.class), mock(Neo4JService.class), dataSecurity, mock(SubscriptionRepository.class), validator, policyEnforcement, mock(DatasetClosureService.class), + mock(IngestQuotaService.class), mock(TenantLimitsService.class), new EdgeMapper(nodeRepository, relationshipTypeRepository, mock(RelationshipTypeService.class)), new ai.intellistream.datahub.api.services.node.NodeUpdateService( nodeRepository, dataSetRepository, dataSecurity, labelService, nodeService, policyEnforcement), diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceEdgeAccessTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceEdgeAccessTest.java index 3f4f0311..b2a27df6 100644 --- a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceEdgeAccessTest.java +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceEdgeAccessTest.java @@ -102,7 +102,8 @@ class ResourceServiceEdgeAccessTest { relationshipTypeRepository, relationshipTypeService, eventPublisher, graphOutbox, neo4JService, dataSecurity, subscriptionRepository, validator, policyEnforcement, datasetClosureService, - new ai.intellistream.datahub.api.edge.EdgeMapper(nodeRepository, relationshipTypeRepository, relationshipTypeService), + mock(IngestQuotaService.class), mock(TenantLimitsService.class), + new ai.intellistream.datahub.api.edge.EdgeMapper(nodeRepository, relationshipTypeRepository, relationshipTypeService), new ai.intellistream.datahub.api.services.node.NodeUpdateService( nodeRepository, dataSetRepository, dataSecurity, labelService, nodeService, policyEnforcement), mock(ai.intellistream.datahub.api.policy.NamingPolicyResolver.class)); diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceEdgeRulesTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceEdgeRulesTest.java index a1f28e3f..a7df9179 100644 --- a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceEdgeRulesTest.java +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceEdgeRulesTest.java @@ -58,7 +58,8 @@ class ResourceServiceEdgeRulesTest { mock(DataSecurity.class), mock(SubscriptionRepository.class), mock(Validator.class), mock(PolicyEnforcement.class), mock(DatasetClosureService.class), - new ai.intellistream.datahub.api.edge.EdgeMapper(nodeRepository, mock(RelationshipTypeRepository.class), relationshipTypeService), + mock(IngestQuotaService.class), mock(TenantLimitsService.class), + new ai.intellistream.datahub.api.edge.EdgeMapper(nodeRepository, mock(RelationshipTypeRepository.class), relationshipTypeService), new ai.intellistream.datahub.api.services.node.NodeUpdateService( mock(NodeRepository.class), mock(DataSetRepository.class), mock(DataSecurity.class), mock(LabelService.class), mock(NodeService.class), mock(PolicyEnforcement.class)), diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceFetchNearestIdentifierTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceFetchNearestIdentifierTest.java new file mode 100644 index 00000000..3da628af --- /dev/null +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceFetchNearestIdentifierTest.java @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.services; + +import ai.intellistream.datahub.api.messaging.outbox.GraphOutbox; +import ai.intellistream.datahub.api.datasecurity.DataSecurity; +import ai.intellistream.datahub.api.datasecurity.DatasetClosureService; +import ai.intellistream.datahub.api.policy.PolicyEnforcement; +import ai.intellistream.datahub.asset.ResourceNetwork; +import ai.intellistream.datahub.errors.ObjectNotFoundException; +import ai.intellistream.datahub.jpa.domains.AssetEntity; +import ai.intellistream.datahub.jpa.domains.NodeEntity; +import ai.intellistream.datahub.models.FetchNearestResourcesForm; +import ai.intellistream.datahub.repositories.node.DataSetRepository; +import ai.intellistream.datahub.repositories.node.EdgeRepository; +import ai.intellistream.datahub.repositories.node.NodeRepository; +import ai.intellistream.datahub.repositories.node.RelationshipTypeRepository; +import ai.intellistream.datahub.repositories.subscription.SubscriptionRepository; +import ai.intellistream.datahub.services.LabelService; +import ai.intellistream.datahub.services.Neo4JService; +import ai.intellistream.datahub.services.NodeService; +import ai.intellistream.datahub.services.RelationshipTypeService; +import jakarta.persistence.EntityManager; +import jakarta.validation.Validator; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.context.ApplicationEventPublisher; + +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * {@code POST /resources/fetch-nearest} accepts an {@code externalId} as well as an {@code id}. + * + *

The form has always declared both and its {@code @OneIdNotNull} validator has always accepted + * either, but the endpoint only ever read {@code id}, so an externalId-only request — a valid one, + * by the API's own contract — reached {@code findById(null)} and came back as a 500. That is worse + * than a plain rejection: it tells the caller the server is broken when their request was fine. + * + *

{@code fetch-related} has resolved both identifiers all along, so this also stops two sibling + * endpoints disagreeing about what an identifier is. + */ +class ResourceServiceFetchNearestIdentifierTest { + + private final NodeRepository nodeRepository = mock(NodeRepository.class); + private final Neo4JService neo4JService = mock(Neo4JService.class); + private final DataSecurity dataSecurity = mock(DataSecurity.class); + + private final ResourceService service = new ResourceService( + mock(EntityManager.class), nodeRepository, mock(NodeService.class), + mock(EdgeRepository.class), + mock(RelationshipTypeRepository.class), mock(RelationshipTypeService.class), + mock(ApplicationEventPublisher.class), mock(GraphOutbox.class), neo4JService, + dataSecurity, + mock(SubscriptionRepository.class), mock(Validator.class), + mock(PolicyEnforcement.class), mock(DatasetClosureService.class), + mock(IngestQuotaService.class), mock(TenantLimitsService.class), + new ai.intellistream.datahub.api.edge.EdgeMapper( + nodeRepository, mock(RelationshipTypeRepository.class), mock(RelationshipTypeService.class)), + new ai.intellistream.datahub.api.services.node.NodeUpdateService( + nodeRepository, mock(DataSetRepository.class), dataSecurity, + mock(LabelService.class), mock(NodeService.class), mock(PolicyEnforcement.class)), + mock(ai.intellistream.datahub.api.policy.NamingPolicyResolver.class)); + + private static NodeEntity node(long id, String externalId) { + AssetEntity entity = new AssetEntity(); + entity.setId(id); + entity.setExternalId(externalId); + entity.setName(externalId); + return entity; + } + + @Test + @DisplayName("an externalId-only request resolves and traverses") + void resolvesExternalId() { + NodeEntity pump = node(42L, "21-p-101a"); + when(nodeRepository.findByExternalId("21-p-101a")).thenReturn(pump); + when(nodeRepository.findById(42L)).thenReturn(Optional.of(pump)); + when(neo4JService.fetchNearestNodesByEndLabel(eq(42L), anyList(), any(), any(), anyList())) + .thenReturn(new ResourceNetwork(Set.of(), Set.of(), Set.of())); + + FetchNearestResourcesForm form = new FetchNearestResourcesForm(); + form.setExternalId("21-p-101a"); + form.setEndLabels(List.of("TIMESERIES")); + + service.fetchNearestRelatedResources(form); + + verify(neo4JService).fetchNearestNodesByEndLabel(eq(42L), eq(List.of("TIMESERIES")), + eq(10), any(), anyList()); + verify(dataSecurity).assertCanRead(pump); + } + + @Test + @DisplayName("an unknown externalId is a 404, not a 500") + void unknownExternalIdIsNotFound() { + when(nodeRepository.findByExternalId("does-not-exist")).thenReturn(null); + + FetchNearestResourcesForm form = new FetchNearestResourcesForm(); + form.setExternalId("does-not-exist"); + form.setEndLabels(List.of("TIMESERIES")); + + assertThatThrownBy(() -> service.fetchNearestRelatedResources(form)) + .isInstanceOf(ObjectNotFoundException.class) + .hasMessageContaining("does-not-exist"); + } + + @Test + @DisplayName("a numeric id still short-circuits the lookup") + void numericIdSkipsResolution() { + NodeEntity pump = node(42L, "21-p-101a"); + when(nodeRepository.findById(42L)).thenReturn(Optional.of(pump)); + when(neo4JService.fetchNearestNodesByEndLabel(eq(42L), anyList(), any(), any(), anyList())) + .thenReturn(new ResourceNetwork(Set.of(), Set.of(), Set.of())); + + FetchNearestResourcesForm form = new FetchNearestResourcesForm(); + form.setId(42L); + form.setEndLabels(List.of("TIMESERIES")); + + service.fetchNearestRelatedResources(form); + + verify(nodeRepository, org.mockito.Mockito.never()).findByExternalId(any()); + } +} diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceNodeTypeAclTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceNodeTypeAclTest.java index 3edb4622..17f18ad1 100644 --- a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceNodeTypeAclTest.java +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/ResourceServiceNodeTypeAclTest.java @@ -87,6 +87,7 @@ class ResourceServiceNodeTypeAclTest { entityManager, nodeRepository, nodeService, edgeRepository, relationshipTypeRepository, relationshipTypeService, eventPublisher, graphOutbox, neo4JService, dataSecurity, subscriptionRepository, validator, policyEnforcement, datasetClosureService, + mock(IngestQuotaService.class), mock(TenantLimitsService.class), new ai.intellistream.datahub.api.edge.EdgeMapper(nodeRepository, relationshipTypeRepository, relationshipTypeService), new ai.intellistream.datahub.api.services.node.NodeUpdateService( nodeRepository, dataSetRepository, dataSecurity, labelService, nodeService, policyEnforcement), diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/SearchFilterNarrowingTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/SearchFilterNarrowingTest.java index 85762e5a..2003c0d5 100644 --- a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/SearchFilterNarrowingTest.java +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/SearchFilterNarrowingTest.java @@ -146,11 +146,12 @@ entityManager, mock(NodeRepository.class), mock(NodeService.class), mock(EdgeRep dataSecurity, mock(SubscriptionRepository.class), mock(Validator.class), mock(PolicyEnforcement.class), closure, + mock(IngestQuotaService.class), mock(TenantLimitsService.class), new ai.intellistream.datahub.api.edge.EdgeMapper(mock(NodeRepository.class), mock(RelationshipTypeRepository.class), mock(RelationshipTypeService.class)), - new ai.intellistream.datahub.api.services.node.NodeUpdateService( - mock(NodeRepository.class), mock(DataSetRepository.class), mock(DataSecurity.class), - mock(LabelService.class), mock(NodeService.class), mock(PolicyEnforcement.class)), - mock(ai.intellistream.datahub.api.policy.NamingPolicyResolver.class)); + new ai.intellistream.datahub.api.services.node.NodeUpdateService( + mock(NodeRepository.class), mock(DataSetRepository.class), mock(DataSecurity.class), + mock(LabelService.class), mock(NodeService.class), mock(PolicyEnforcement.class)), + mock(ai.intellistream.datahub.api.policy.NamingPolicyResolver.class)); private SearchBody searchFor(String query, ResourceFilter filter) { SearchBody form = new SearchBody<>(); diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/TenantLimitsServiceTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/TenantLimitsServiceTest.java new file mode 100644 index 00000000..995d753b --- /dev/null +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/TenantLimitsServiceTest.java @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +package ai.intellistream.datahub.api.services; + +import ai.intellistream.datahub.api.config.LimitsProperties; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.ResultSetExtractor; + +import java.sql.ResultSet; +import java.time.Duration; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Resolving a tenant's limits: overrides win, NULL inherits, and nothing about a missing or broken + * row is allowed to refuse a request. + */ +class TenantLimitsServiceTest { + + private static final String TENANT = "acme"; + + private final LimitsProperties defaults = new LimitsProperties(); + private final JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class); + private final AtomicLong now = new AtomicLong(0); + + private TenantLimitsService service(Duration ttl) { + return new TenantLimitsService(defaults, jdbcTemplate, ttl, now::get); + } + + /** + * A one-row result where only {@code set} columns have a value and everything else is NULL. + * {@code wasNull()} has to answer for the column just read, which is what tells an override + * apart from an inherited default. + */ + @SuppressWarnings("unchecked") + private void rowWith(Map set) throws Exception { + ResultSet rs = mock(ResultSet.class); + AtomicBoolean lastWasNull = new AtomicBoolean(true); + + when(rs.next()).thenReturn(true); + when(rs.getInt(anyString())).thenAnswer(inv -> { + Number value = set.get(inv.getArgument(0)); + lastWasNull.set(value == null); + return value == null ? 0 : value.intValue(); + }); + when(rs.getLong(anyString())).thenAnswer(inv -> { + Number value = set.get(inv.getArgument(0)); + lastWasNull.set(value == null); + return value == null ? 0L : value.longValue(); + }); + when(rs.wasNull()).thenAnswer(inv -> lastWasNull.get()); + + when(jdbcTemplate.query(anyString(), any(ResultSetExtractor.class))) + .thenAnswer(inv -> ((ResultSetExtractor) inv.getArgument(1)).extractData(rs)); + } + + @Test + void anOverrideWinsAndTheRestOfTheRowStillInherits() throws Exception { + rowWith(Map.of("write_per_minute_per_tenant", 42, "max_resources", 1000L)); + + TenantLimits limits = service(Duration.ofMinutes(5)).forTenant(TENANT); + + assertThat(limits.writePerMinutePerTenant()).isEqualTo(42); + assertThat(limits.maxResources()).isEqualTo(1000L); + assertThat(limits.readPerMinutePerTenant()) + .as("a NULL column inherits the deployment default") + .isEqualTo(defaults.getRate().getReadPerMinutePerTenant()); + } + + @Test + void aZeroOverrideMeansUnlimitedRatherThanInherit() throws Exception { + // This is how a limit gets lifted for a customer who asked: set the column to 0. + rowWith(Map.of("write_per_minute_per_tenant", 0)); + + TenantLimits limits = service(Duration.ofMinutes(5)).forTenant(TENANT); + + assertThat(limits.writePerMinutePerTenant()).isZero(); + assertThat(TenantLimits.unlimited(limits.writePerMinutePerTenant())).isTrue(); + } + + @Test + @SuppressWarnings("unchecked") + void aMissingRowInheritsEveryDefault() { + when(jdbcTemplate.query(anyString(), any(ResultSetExtractor.class))).thenReturn(null); + + TenantLimits limits = service(Duration.ofMinutes(5)).forTenant(TENANT); + + assertThat(limits.writePerMinutePerTenant()) + .isEqualTo(defaults.getRate().getWritePerMinutePerTenant()); + assertThat(limits.readPerMinutePerUser()) + .isEqualTo(defaults.getRate().getReadPerMinutePerUser()); + } + + @Test + @SuppressWarnings("unchecked") + void anUnreadableTableFallsBackToDefaultsRatherThanFailing() { + // The rate-limit filter runs before tenant provisioning, so a first-touch request reaches + // this before the schema exists. That must not refuse the request. + when(jdbcTemplate.query(anyString(), any(ResultSetExtractor.class))) + .thenThrow(new org.springframework.jdbc.BadSqlGrammarException( + "read", "SELECT * FROM tenant_limits WHERE id = 1", new java.sql.SQLException())); + + TenantLimits limits = service(Duration.ofMinutes(5)).forTenant(TENANT); + + assertThat(limits.writePerMinutePerTenant()) + .isEqualTo(defaults.getRate().getWritePerMinutePerTenant()); + } + + @Test + @SuppressWarnings("unchecked") + void aResolvedRowIsCachedUntilTheTtlExpires() { + when(jdbcTemplate.query(anyString(), any(ResultSetExtractor.class))).thenReturn(null); + TenantLimitsService service = service(Duration.ofMinutes(5)); + + service.forTenant(TENANT); + now.addAndGet(Duration.ofMinutes(4).toMillis()); + service.forTenant(TENANT); + + verify(jdbcTemplate, times(1)).query(anyString(), any(ResultSetExtractor.class)); + } + + @Test + @SuppressWarnings("unchecked") + void pastTheTtlTheRowIsReadAgainSoAnUpdatePropagates() { + when(jdbcTemplate.query(anyString(), any(ResultSetExtractor.class))).thenReturn(null); + TenantLimitsService service = service(Duration.ofMinutes(5)); + + service.forTenant(TENANT); + now.addAndGet(Duration.ofMinutes(5).toMillis() + 1); + service.forTenant(TENANT); + + // This re-read is the whole propagation mechanism: raising a limit is an UPDATE, and every + // instance picks it up within the TTL with no restart. + verify(jdbcTemplate, times(2)).query(anyString(), any(ResultSetExtractor.class)); + } + + @Test + @SuppressWarnings("unchecked") + void tenantsAreCachedIndependently() { + when(jdbcTemplate.query(anyString(), any(ResultSetExtractor.class))).thenReturn(null); + TenantLimitsService service = service(Duration.ofMinutes(5)); + + service.forTenant("tenant-a"); + service.forTenant("tenant-b"); + service.forTenant("tenant-a"); + + verify(jdbcTemplate, times(2)).query(anyString(), any(ResultSetExtractor.class)); + } +} diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/TimeseriesServiceInsertDatapointsTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/TimeseriesServiceInsertDatapointsTest.java index b2561203..48349f5b 100644 --- a/datahub-api/src/test/java/ai/intellistream/datahub/api/services/TimeseriesServiceInsertDatapointsTest.java +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/services/TimeseriesServiceInsertDatapointsTest.java @@ -1,8 +1,10 @@ // SPDX-License-Identifier: AGPL-3.0-or-later package ai.intellistream.datahub.api.services; +import ai.intellistream.datahub.api.controllers.errors.BadRequestException; import ai.intellistream.datahub.api.datasecurity.DataSecurity; import ai.intellistream.datahub.api.responses.DataWrapper; +import ai.intellistream.datahub.models.validation.FieldLimits; import ai.intellistream.datahub.api.responses.DataWrapperBin; import ai.intellistream.datahub.api.responses.DatapointString; import ai.intellistream.datahub.api.responses.DatapointsCollection; @@ -11,6 +13,9 @@ import ai.intellistream.datahub.repositories.node.TimeseriesRepository; import ai.intellistream.datahub.tenant.TenantContext; import ai.intellistream.datahub.services.ValkeyService; +import jakarta.validation.ConstraintViolation; +import jakarta.validation.ConstraintViolationException; +import jakarta.validation.Validator; import org.apache.pulsar.client.api.Producer; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -36,6 +41,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -66,6 +72,10 @@ class TimeseriesServiceInsertDatapointsTest { @Mock private Producer allDatapointProducer; @Mock private LiveIngestCounter datapointIngestCounter; @Mock private TransactionTemplate transactionTemplate; + @Mock private IngestQuotaService ingestQuota; + // insertDatapoints re-validates, because the timeseries_send_datapoint MCP tool reaches it + // without passing a controller. A mock returns no violations, so it waves the fixtures through. + @Mock private Validator validator; @InjectMocks private TimeseriesService timeseriesService; @@ -283,4 +293,55 @@ void emptyCollectionDoesNotTouchTheLatestValueCache() throws Exception { // NullPointerException depending on whether the key already had a value. verify(valkeyService, never()).setLatestDatapoint(anyString(), any(DatapointString.class)); } + + // ---- limits ----------------------------------------------------------- + + @Test + @DisplayName("Bean constraints are enforced here, not only at the controller (the MCP path)") + @SuppressWarnings("unchecked") + void constraintViolationsRefuseTheRequestBeforeAnythingIsPublished() throws Exception { + // timeseries_send_datapoint calls this method directly, so a caller that never touches a + // controller would otherwise face no size limits at all. + ConstraintViolation violation = mock(ConstraintViolation.class); + when(validator.validate(any(DataWrapper.class))).thenReturn(java.util.Set.of(violation)); + + assertThrows(ConstraintViolationException.class, () -> timeseriesService.insertDatapoints( + request(collection("pump-1", point("2026-08-21T10:00:00Z", "1.5"))))); + + verify(allDatapointProducer, never()).send(any(DataWrapperBin.class)); + } + + @Test + @DisplayName("A TEXT series takes a smaller batch than a numeric one") + void textCollectionsAreCappedTighterThanNumericOnes() throws Exception { + known("status-1", 1L, "MIXED"); + + DatapointString[] tooMany = new DatapointString[FieldLimits.TEXT_DATAPOINTS_PER_COLLECTION_MAX + 1]; + for (int i = 0; i < tooMany.length; i++) { + tooMany[i] = point("2026-08-21T10:00:00Z", "1.0"); + } + + // A text batch is the one shape that can approach Pulsar's per-message ceiling. The cap + // cannot live on the DTO: it depends on the value type, which is only known once the + // series has been resolved. + assertThrows(BadRequestException.class, + () -> timeseriesService.insertDatapoints(request(collection("status-1", tooMany)))); + + verify(allDatapointProducer, never()).send(any(DataWrapperBin.class)); + } + + @Test + @DisplayName("A numeric series still takes a batch larger than the text cap") + void numericCollectionsKeepTheLargerCap() throws Exception { + known("pump-1", 1L, "FLOAT"); + + DatapointString[] points = new DatapointString[FieldLimits.TEXT_DATAPOINTS_PER_COLLECTION_MAX + 1]; + for (int i = 0; i < points.length; i++) { + points[i] = point("2026-08-21T10:00:00Z", "1.0"); + } + + timeseriesService.insertDatapoints(request(collection("pump-1", points))); + + verify(allDatapointProducer).send(any(DataWrapperBin.class)); + } } diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/websocket/DatapointListenWebSocketHandlerIT.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/websocket/DatapointListenWebSocketHandlerIT.java index 94edc8bd..9b18aea3 100644 --- a/datahub-api/src/test/java/ai/intellistream/datahub/api/websocket/DatapointListenWebSocketHandlerIT.java +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/websocket/DatapointListenWebSocketHandlerIT.java @@ -51,7 +51,19 @@ void setUp() { TopicNames topicNames = topicNames(); StreamAccessAuthorizer authorizer = new StreamAccessAuthorizer( mock(TimeseriesRepository.class), mock(SubscriptionRepository.class), testGroupsResolver()); - handler = new DatapointListenWebSocketHandler(pulsarClient, topicNames, jwtDecoder, JSON, authorizer); + handler = new DatapointListenWebSocketHandler(pulsarClient, topicNames, jwtDecoder, JSON, + authorizer, allowAllLimiter()); + } + + /** + * A limiter that allows every connection: its limits service answers null, which the limiter + * treats as "no ceiling known". These tests are about the streaming protocol, not the caps. + */ + private static WebSocketConnectionLimiter allowAllLimiter() { + return new WebSocketConnectionLimiter( + mock(ai.intellistream.datahub.api.services.TenantLimitsService.class), + mock(ai.intellistream.datahub.services.ValkeyService.class), + new ai.intellistream.datahub.api.config.LimitsProperties()); } @AfterEach diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/api/websocket/SubscriptionWebSocketHandlerIT.java b/datahub-api/src/test/java/ai/intellistream/datahub/api/websocket/SubscriptionWebSocketHandlerIT.java index 7911f425..c7d2b086 100644 --- a/datahub-api/src/test/java/ai/intellistream/datahub/api/websocket/SubscriptionWebSocketHandlerIT.java +++ b/datahub-api/src/test/java/ai/intellistream/datahub/api/websocket/SubscriptionWebSocketHandlerIT.java @@ -60,7 +60,19 @@ void setUp() { TopicNames topicNames = topicNames(); StreamAccessAuthorizer authorizer = new StreamAccessAuthorizer( mock(TimeseriesRepository.class), subscriptionRepository, testGroupsResolver()); - handler = new SubscriptionWebSocketHandler(pulsarClient, topicNames, subscriptionRepository, JSON, authorizer); + handler = new SubscriptionWebSocketHandler(pulsarClient, topicNames, subscriptionRepository, JSON, + authorizer, allowAllLimiter()); + } + + /** + * A limiter that allows every connection: its limits service answers null, which the limiter + * treats as "no ceiling known". These tests are about the streaming protocol, not the caps. + */ + private static WebSocketConnectionLimiter allowAllLimiter() { + return new WebSocketConnectionLimiter( + mock(ai.intellistream.datahub.api.services.TenantLimitsService.class), + mock(ai.intellistream.datahub.services.ValkeyService.class), + new ai.intellistream.datahub.api.config.LimitsProperties()); } @AfterEach diff --git a/datahub-api/src/test/java/ai/intellistream/datahub/config/SecurityFilterChainTest.java b/datahub-api/src/test/java/ai/intellistream/datahub/config/SecurityFilterChainTest.java index e7d70830..5b3d0c23 100644 --- a/datahub-api/src/test/java/ai/intellistream/datahub/config/SecurityFilterChainTest.java +++ b/datahub-api/src/test/java/ai/intellistream/datahub/config/SecurityFilterChainTest.java @@ -12,7 +12,10 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.security.web.FilterChainProxy; +import org.springframework.security.web.SecurityFilterChain; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpStatus; import org.springframework.security.oauth2.jwt.Jwt; @@ -101,6 +104,10 @@ private static Stream securedPaths() { @Value("${local.management.port}") private int managementPort; + /** The assembled chain, for asserting filter order rather than only filter behaviour. */ + @Autowired + private FilterChainProxy securityFilterChainProxy; + @MockitoBean private JwtDecoder jwtDecoder; @@ -245,6 +252,25 @@ void actuatorIsAbsentFromTheApplicationPort() { .isNotEqualTo(HttpStatus.OK.value()); } + @Test + @DisplayName("Rate limiting runs after authorization and before tenant provisioning") + void rateLimitingSitsBetweenAuthorizationAndTenantProvisioning() { + List filters = securityFilterChainProxy.getFilterChains().stream() + .flatMap(chain -> ((SecurityFilterChain) chain).getFilters().stream()) + .map(filter -> filter.getClass().getSimpleName()) + .toList(); + + int authorization = filters.indexOf("AuthorizationFilter"); + int rateLimit = filters.indexOf("RateLimitFilter"); + int provisioning = filters.indexOf("TenantProvisioningFilter"); + + assertThat(rateLimit).as("the limiter must be in the chain at all").isPositive(); + // After authorization, because the budget is charged to an authenticated identity. Before + // provisioning, so an over-budget caller costs no Vault lookup and no Flyway check. + assertThat(rateLimit).isGreaterThan(authorization); + assertThat(rateLimit).isLessThan(provisioning); + } + // ---- helpers ----------------------------------------------------------------------------- /** diff --git a/datahub-commons/src/main/java/ai/intellistream/datahub/api/responses/swaggerdto/DatapointsCollectionDataWrapper.java b/datahub-commons/src/main/java/ai/intellistream/datahub/api/responses/swaggerdto/DatapointsCollectionDataWrapper.java index 1af34bc1..26e05657 100644 --- a/datahub-commons/src/main/java/ai/intellistream/datahub/api/responses/swaggerdto/DatapointsCollectionDataWrapper.java +++ b/datahub-commons/src/main/java/ai/intellistream/datahub/api/responses/swaggerdto/DatapointsCollectionDataWrapper.java @@ -2,9 +2,11 @@ package ai.intellistream.datahub.api.responses.swaggerdto; import ai.intellistream.datahub.api.responses.DatapointsCollection; +import ai.intellistream.datahub.models.validation.FieldLimits; import com.fasterxml.jackson.annotation.JsonInclude; import tools.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.Size; import java.util.ArrayList; import java.util.Collection; @@ -13,6 +15,7 @@ public class DatapointsCollectionDataWrapper { @JacksonXmlElementWrapper(useWrapping = false) + @Size(max = FieldLimits.BATCH_ITEMS_MAX) private Collection items = new ArrayList<>(); @JsonInclude(JsonInclude.Include.ALWAYS) diff --git a/datahub-commons/src/main/java/ai/intellistream/datahub/api/responses/swaggerdto/UpdateEventDataWrapper.java b/datahub-commons/src/main/java/ai/intellistream/datahub/api/responses/swaggerdto/UpdateEventDataWrapper.java index b70e2b9e..85d9f441 100644 --- a/datahub-commons/src/main/java/ai/intellistream/datahub/api/responses/swaggerdto/UpdateEventDataWrapper.java +++ b/datahub-commons/src/main/java/ai/intellistream/datahub/api/responses/swaggerdto/UpdateEventDataWrapper.java @@ -2,8 +2,10 @@ package ai.intellistream.datahub.api.responses.swaggerdto; import ai.intellistream.datahub.models.UpdateEventForm; +import ai.intellistream.datahub.models.validation.FieldLimits; import com.fasterxml.jackson.annotation.JsonInclude; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.Size; import tools.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import java.util.ArrayList; @@ -13,6 +15,7 @@ public class UpdateEventDataWrapper { @JacksonXmlElementWrapper(useWrapping = false) + @Size(max = FieldLimits.BATCH_ITEMS_MAX) private Collection items = new ArrayList<>(); @JsonInclude(JsonInclude.Include.ALWAYS) diff --git a/datahub-commons/src/test/java/ai/intellistream/datahub/config/VaultClientFactoryTlsTest.java b/datahub-commons/src/test/java/ai/intellistream/datahub/config/VaultClientFactoryTlsTest.java index 441834c9..a6f9d9e0 100644 --- a/datahub-commons/src/test/java/ai/intellistream/datahub/config/VaultClientFactoryTlsTest.java +++ b/datahub-commons/src/test/java/ai/intellistream/datahub/config/VaultClientFactoryTlsTest.java @@ -13,6 +13,7 @@ import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLHandshakeException; import javax.net.ssl.TrustManagerFactory; import java.io.IOException; import java.io.InputStream; @@ -136,14 +137,23 @@ void vaultDriverUsesTheSameSslSettings() throws Exception { assertThat(health.getSealed()).isFalse(); } + /** + * A client with no certificate gets nothing back, however the rejection is delivered. + * + *

The assertion is deliberately only "it failed". Under TLS 1.3 the client finishes its side + * of the handshake before the server has looked at the client certificate it never sent, so the + * refusal usually arrives on the first read, as {@code IOException: HTTP/1.1 header parser + * received no bytes}, rather than as an {@link SSLHandshakeException}. Both happen, and which + * one the client sees is a race: requiring the SSL-specific shape failed this test in roughly + * nine runs out of ten. What the mutual-TLS setting has to guarantee is that an uncertificated + * client is refused, and {@link #keystoreAndTruststoreCompleteTheMutualTlsHandshake()} on the + * same server proves the failure here is the missing certificate rather than an unreachable + * server. + */ @Test void withoutAClientCertificateTheServerRejectsTheHandshake() { var ssl = VaultClientFactory.sslContext(tls(null, trustStore)).orElseThrow(); - // The shape of the failure depends on the TLS version: 1.2 rejects the handshake itself, - // while 1.3 sends the client certificate after the handshake completes, so the server's - // refusal arrives as an ordinary IOException on the response read. Either way the request - // does not succeed, which is what this asserts. assertThatThrownBy(() -> health(ssl)).isInstanceOf(IOException.class); } diff --git a/datahub-console/scripts/fa-subset.py b/datahub-console/scripts/fa-subset.py index 9751d80b..da6c1ad1 100644 --- a/datahub-console/scripts/fa-subset.py +++ b/datahub-console/scripts/fa-subset.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-or-later """Report (or regenerate) the Font Awesome subset embedded at the top of all.css. The console renders ~45 of Font Awesome 6 Free Solid's 1950 glyphs, so all.css carries a diff --git a/datahub-console/src/main/java/ai/intellistream/dhconsole/api/DatahubApi.java b/datahub-console/src/main/java/ai/intellistream/dhconsole/api/DatahubApi.java index a39a5efa..0b3dd73c 100644 --- a/datahub-console/src/main/java/ai/intellistream/dhconsole/api/DatahubApi.java +++ b/datahub-console/src/main/java/ai/intellistream/dhconsole/api/DatahubApi.java @@ -54,20 +54,13 @@ public interface DatahubApi { @RequestLine("POST /resources/filter") DataWrapper filter(ResourceRetreiver apiReqData); - @RequestLine("GET /resources/{id}") - DataWrapper get(@Param("id") Long id); - @RequestLine("DELETE /resources/delete") GraphDataWrapper deleteResource(DataWrapper apiReqData); @RequestLine("POST /resources/search") DataWrapper searchResource(SearchBody form); - @RequestLine("GET /edges/{id}") - DataWrapper getEdgeById(@Param("id") Long id); - @RequestLine("POST /edges/byids") - GraphDataWrapper getEdgeWithNodesById(DataWrapper apiReqData); @RequestLine("POST /timeseries/data/list") DataWrapper> retrieveDatapoints(DataRetriever apiReqData); @@ -117,11 +110,6 @@ public interface DatahubApi { @RequestLine("GET /policies/{policyNodeId}") DataWrapper getPolicyById(@Param("policyNodeId") Long policyNodeId); - @RequestLine("POST /policies/apply-template?policyNodeId={policyNodeId}&templateId={templateId}") - DataWrapper applyPolicyTemplate( - @Param("policyNodeId") Long policyNodeId, - @Param("templateId") Long templateId - ); @RequestLine("POST /policies/create") DataWrapper createPolicies(DataWrapper wrapper); @@ -132,13 +120,7 @@ DataWrapper applyPolicyTemplate( @RequestLine("DELETE /policies/delete") void deletePolicies(DataWrapper wrapper); - @RequestLine("GET /governance/templates") - DataWrapper getGovernanceTemplates(); - @RequestLine("GET /governance/templates/{templateId}") - DataWrapper getGovernanceTemplateById( - @Param("templateId") Long templateId - ); // TIMESERIES @RequestLine("GET /timeseries") @@ -174,6 +156,4 @@ DataWrapper getGovernanceTemplateById( @RequestLine("GET /files/list{path}") DataWrapper listDirectory(@Param("path") String path); - @RequestLine("GET /files/download/{id}") - ResponseEntity download(@Param("id") String id); } diff --git a/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/GovernanceController.java b/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/GovernanceController.java deleted file mode 100644 index f89fe373..00000000 --- a/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/GovernanceController.java +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later -package ai.intellistream.dhconsole.controllers; - -import ai.intellistream.dhconsole.api.DatahubApi; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Controller; -import org.springframework.ui.Model; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; - -@Slf4j -@Controller -@RequestMapping("/governance") -@RequiredArgsConstructor -public class GovernanceController { - - private final DatahubApi datahubApi; - - @GetMapping("") - public String index(Model model) { - model.addAttribute("templates", datahubApi.getGovernanceTemplates().getItems()); - return "governance/index"; // Thymeleaf page - } -} diff --git a/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/DataSetApiController.java b/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/DataSetApiController.java index 0765c3d9..f0c4f15d 100644 --- a/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/DataSetApiController.java +++ b/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/DataSetApiController.java @@ -68,10 +68,4 @@ public ResponseEntity delete(@PathVariable long id){ } return ResponseEntity.noContent().build(); } - - @RequestMapping(value = {"/policies"}, method = RequestMethod.GET) - public ResponseEntity policies(){ - DataWrapper data = datahubApi.getPolicies(); - return new ResponseEntity<>(data, HttpStatus.OK); - } } diff --git a/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/GovernanceApiController.java b/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/GovernanceApiController.java deleted file mode 100644 index 808a98fd..00000000 --- a/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/GovernanceApiController.java +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later -package ai.intellistream.dhconsole.controllers.api; - -import ai.intellistream.datahub.api.responses.DataWrapper; -import ai.intellistream.datahub.models.GovernanceTemplateDTO; -import ai.intellistream.dhconsole.api.DatahubApi; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; - -@Slf4j -@RestController -@RequestMapping("/api/governance") -@RequiredArgsConstructor -public class GovernanceApiController { - - private final DatahubApi datahubApi; - - @GetMapping(value = "/templates", produces = "application/json") - public ResponseEntity> listTemplates() { - return ResponseEntity.ok(datahubApi.getGovernanceTemplates()); - } - - @GetMapping(value = "/templates/{templateId}", produces = "application/json") - public ResponseEntity> getTemplateById(@PathVariable Long templateId) { - DataWrapper data = datahubApi.getGovernanceTemplateById(templateId); - return ResponseEntity.ok(data); - } -} diff --git a/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/LabelApiController.java b/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/LabelApiController.java index f3d4bd2b..fa75e6b8 100644 --- a/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/LabelApiController.java +++ b/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/LabelApiController.java @@ -94,8 +94,4 @@ public ResponseEntity update(@RequestBody @Valid LabelForm form){ .orElseGet(() -> new ResponseEntity<>(form, HttpStatus.BAD_REQUEST)); } - @RequestMapping(value = {"/delete"}, method = {RequestMethod.POST, RequestMethod.DELETE}) - public ResponseEntity delete(){ - return ResponseEntity.noContent().build(); - } } diff --git a/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/PolicyApiController.java b/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/PolicyApiController.java index e7e94931..e8b13d6b 100644 --- a/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/PolicyApiController.java +++ b/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/PolicyApiController.java @@ -27,14 +27,6 @@ public ResponseEntity> listPolicies() { return ResponseEntity.ok(datahubApi.getPolicies()); } - /** - * List available policy types (IS_WRITE_PROTECTED, MASKING_POLICY...) - */ - @GetMapping("/types") - public ResponseEntity> listPolicyTypes() { - return ResponseEntity.ok(datahubApi.getPolicyTypes()); - } - /** * Load a specific policy node. */ diff --git a/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/ResourceApiController.java b/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/ResourceApiController.java index c3ba6025..59ceb590 100644 --- a/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/ResourceApiController.java +++ b/datahub-console/src/main/java/ai/intellistream/dhconsole/controllers/api/ResourceApiController.java @@ -63,16 +63,6 @@ public ResponseEntity fetchRelatedNodes(@RequestBody RelatedResourcesForm for return new ResponseEntity<>(resourceNetwork, HttpStatus.OK); } - @RequestMapping(value = {"/byids"}, - method = RequestMethod.POST, - produces = {MediaType.APPLICATION_JSON_VALUE}, - consumes = {MediaType.APPLICATION_JSON_VALUE} - ) - public ResponseEntity byIds(@RequestBody DataWrapper apiReqData){ - DataWrapper resources = this.datahubApi.byIds(apiReqData); - return new ResponseEntity<>(resources, HttpStatus.OK); - } - @RequestMapping(value = {"/save"}, method = RequestMethod.POST, produces = {MediaType.APPLICATION_JSON_VALUE}, diff --git a/datahub-console/src/main/java/ai/intellistream/dhconsole/services/LabelApiService.java b/datahub-console/src/main/java/ai/intellistream/dhconsole/services/LabelApiService.java deleted file mode 100644 index 0fbc397e..00000000 --- a/datahub-console/src/main/java/ai/intellistream/dhconsole/services/LabelApiService.java +++ /dev/null @@ -1,8 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later -package ai.intellistream.dhconsole.services; - -import org.springframework.stereotype.Component; - -@Component -public class LabelApiService { -} diff --git a/datahub-console/src/main/resources/i18n/messages.properties b/datahub-console/src/main/resources/i18n/messages.properties index baed7fa8..67cb82e8 100644 --- a/datahub-console/src/main/resources/i18n/messages.properties +++ b/datahub-console/src/main/resources/i18n/messages.properties @@ -739,6 +739,29 @@ error.dataset.read.forbidden=You do not have read access to this data set. Ask a error.permissions.unavailable=Your access could not be checked right now. Wait a moment and try again. error.save.failed=The change could not be saved. Please try again. +# Limit refusals. These are not "your input was wrong", so each says what the limit is and how it +# clears: a rate limit and a daily quota pass on their own, a tenant ceiling is raised by asking. The +# .plain variants are for the paths that only have a status code, with no numbers to fill in. +error.limit.rate=Too many requests in a short time. Try again {0}. +error.limit.rate.plain=Too many requests in a short time. Wait a moment and try again. +error.limit.quota=This organisation has used its ingest allowance for today. Try again {0}. +error.limit.tenant=This organisation has reached its limit of {0} {1}. Contact IntelliStream to have it raised. +error.limit.tenant.plain=This organisation has reached one of its limits. Contact IntelliStream to have it raised. +error.limit.too.large=That is too large to send in one request. The most this endpoint accepts is {0}, so split it into smaller parts. +error.limit.too.large.plain=That is too large to send in one request. Split it into smaller parts. +error.limit.wait.moment=in a moment +error.limit.wait.seconds=in about {0} seconds +error.limit.wait.minutes=in about {0} minutes +error.limit.wait.hours=in about {0} hours +# What the api counted, for the tenant-ceiling message. Anything not listed falls back to the +# server's own English word rather than leaving a gap in the sentence. +error.limit.metric.events=events +error.limit.metric.nodes=objects +error.limit.metric.relationships=relationships +error.limit.metric.datapoints=data points +error.limit.metric.text.datapoints=text data points +error.limit.metric.bytes=ingested bytes + # Chat panel chat.title=Ask your data chat.ask.ai=Ask AI diff --git a/datahub-console/src/main/resources/i18n/messages_nb.properties b/datahub-console/src/main/resources/i18n/messages_nb.properties index 0dc170ac..f7368d37 100644 --- a/datahub-console/src/main/resources/i18n/messages_nb.properties +++ b/datahub-console/src/main/resources/i18n/messages_nb.properties @@ -767,6 +767,29 @@ error.dataset.read.forbidden=Du har ikke lesetilgang til dette datasettet. Konta error.permissions.unavailable=Tilgangene dine kunne ikke sjekkes akkurat nå. Vent litt og prøv igjen. error.save.failed=Endringen kunne ikke lagres. Prøv igjen. +# Avslag på grunn av en grense. Dette er ikke feil i det som ble sendt inn, så hver melding sier hva +# grensen er og hvordan den løser seg: en fartsgrense og en dagskvote går over av seg selv, mens et +# tak for organisasjonen heves ved å spørre. .plain-variantene brukes der bare statuskoden er kjent. +error.limit.rate=For mange forespørsler på kort tid. Prøv igjen {0}. +error.limit.rate.plain=For mange forespørsler på kort tid. Vent litt og prøv igjen. +error.limit.quota=Denne organisasjonen har brukt opp dagens kvote for innlesing. Prøv igjen {0}. +error.limit.tenant=Denne organisasjonen har nådd grensen på {0} {1}. Kontakt IntelliStream for å få den hevet. +error.limit.tenant.plain=Denne organisasjonen har nådd en av grensene sine. Kontakt IntelliStream for å få den hevet. +error.limit.too.large=Dette er for stort til å sendes i en forespørsel. Grensen her er {0}, så del det opp i mindre deler. +error.limit.too.large.plain=Dette er for stort til å sendes i en forespørsel. Del det opp i mindre deler. +error.limit.wait.moment=om et øyeblikk +error.limit.wait.seconds=om omtrent {0} sekunder +error.limit.wait.minutes=om omtrent {0} minutter +error.limit.wait.hours=om omtrent {0} timer +# Hva api-et har talt, brukt i meldingen om taket for organisasjonen. Det som ikke står her faller +# tilbake til api-ets eget engelske ord, slik at setningen fortsatt blir hel. +error.limit.metric.events=hendelser +error.limit.metric.nodes=objekter +error.limit.metric.relationships=relasjoner +error.limit.metric.datapoints=datapunkter +error.limit.metric.text.datapoints=tekstdatapunkter +error.limit.metric.bytes=innleste byte + # Om DataHub (dialogen bak About-oppføringen i brukermenyen). about.license.* er tilbudet om # kildekode etter AGPL paragraf 13, og må fortsatt peke til kodelageret. about.menu=Om DataHub diff --git a/datahub-console/src/main/resources/static/js/app.manifest.js b/datahub-console/src/main/resources/static/js/app.manifest.js index 20c0d36d..0790b8df 100644 --- a/datahub-console/src/main/resources/static/js/app.manifest.js +++ b/datahub-console/src/main/resources/static/js/app.manifest.js @@ -6,7 +6,11 @@ // policy/naming-policy.js is site-wide rather than form-local because two unrelated bundles need it: // the right-form bundle (write responses, inline external-id validation) and the findings queue. // +// limit-errors.js needs $L and getByteSize from application.js, and is needed in turn by the form +// bundle and by the ad-hoc file/upload error paths, so it belongs here rather than in either. +// //= require application.js +//= require limit-errors.js //= require search-dropdown.js //= require enhanced-select.js //= require context-menu.js diff --git a/datahub-console/src/main/resources/static/js/charts/datapoint-stream.js b/datahub-console/src/main/resources/static/js/charts/datapoint-stream.js index 9c75f99d..4756efbb 100644 --- a/datahub-console/src/main/resources/static/js/charts/datapoint-stream.js +++ b/datahub-console/src/main/resources/static/js/charts/datapoint-stream.js @@ -10,7 +10,8 @@ * Usage: * const stream = new DatapointStream({ * onData: points => { ... }, // points: [{externalId, valueType, timestamp, value}] - * onStatus: state => { ... } // state: 'connecting' | 'open' | 'closed' + * onStatus: (state, info) => { ... } // state: 'connecting' | 'open' | 'closed' | 'refused' + * // info: {message} on 'refused' * }); * stream.setInterest(['tsA', 'tsB']); // which timeseries to receive * stream.connect(); @@ -32,6 +33,9 @@ class DatapointStream { this.ws = null; this.active = false; // true between connect() and close() this._reconnectTimer = null; + // Set when the server refused the connection for a limit. Reconnecting on that would hammer + // an endpoint that will refuse every attempt, so the stream stops until the caller retries. + this._refused = false; } // Turn the http(s) api base URL into a ws(s) WebSocket URL for the listen endpoint. @@ -50,6 +54,7 @@ class DatapointStream { connect(){ this.active = true; + this._refused = false; this._open(); } @@ -70,11 +75,25 @@ class DatapointStream { ws.onmessage = ev => { let msg; try { msg = JSON.parse(ev.data); } catch(e){ return; } + // A refusal arrives as one frame and is followed immediately by a close. Record + // it here so onclose knows not to reconnect into the same wall. + if(msg && msg.error && msg.reason === 'websocket-limit-reached'){ + this._refused = true; + this.onStatus('refused', { + message: (window.LimitErrors && window.LimitErrors.fromStatus(429)) + || msg.message + }); + return; + } if(msg && Array.isArray(msg.datapoints) && msg.datapoints.length){ this.onData(msg.datapoints); } }; ws.onclose = () => { + if(this._refused){ + this.onStatus('closed'); + return; + } this.onStatus('closed'); if(this.active) this._scheduleReconnect(); }; diff --git a/datahub-console/src/main/resources/static/js/files-page.js b/datahub-console/src/main/resources/static/js/files-page.js index 6ab28c51..42df074e 100644 --- a/datahub-console/src/main/resources/static/js/files-page.js +++ b/datahub-console/src/main/resources/static/js/files-page.js @@ -366,7 +366,7 @@ Flash.error($L('target.already.exists')); } else { reenable(); - Flash.error($L('update.failed')); + Flash.error(window.LimitErrors.fromStatus(resp.status) || $L('update.failed')); } } @@ -503,7 +503,12 @@ return; } btn.disabled = false; - return r.text().then(msg => flashErr(i18n.restoreFail + (msg ? ': ' + msg : ''))); + // A limit refusal is a whole sentence of its own; anything else keeps the old + // "could not restore: " shape. + return r.text().then(msg => { + const limit = window.LimitErrors.fromStatus(r.status, msg); + flashErr(limit || (i18n.restoreFail + (msg ? ': ' + msg : ''))); + }); }); }).catch(() => { btn.disabled = false; flashErr(i18n.restoreFail); }); } diff --git a/datahub-console/src/main/resources/static/js/limit-errors.js b/datahub-console/src/main/resources/static/js/limit-errors.js new file mode 100644 index 00000000..c4edc837 --- /dev/null +++ b/datahub-console/src/main/resources/static/js/limit-errors.js @@ -0,0 +1,128 @@ +/** + * The api's limit refusals, said in the user's own language. + * + * Four things can now come back that are not "your input was wrong": a per-minute rate limit, a + * daily ingest quota, a lifetime tenant ceiling, and a request body that is too large. The api's own + * `detail` is written for whoever operates the platform, mentions no UI at all, and is never + * translated, so the console rewrites each one and adds the piece that actually helps: how long to + * wait, or that the limit is raised by asking. + * + * These arrive as RFC 9457 problem documents, but a couple of call sites (the raw-body file upload) + * only have a status code and a string, so `fromStatus` covers those without them having to parse. + */ +/** + * The api's per-field caps, mirrored so a form can stop an over-long value where it is typed rather + * than after a round trip that refuses the whole submission. + * + * These duplicate `FieldLimits` in datahub-api-model. The browser copy is a courtesy, not the + * enforcement: the api validates every one of these itself, and a stale value here costs a clearer + * error message, never a hole in the limits. + */ +window.FieldLimits = Object.freeze({ + DESCRIPTION_MAX: 10000, + METADATA_MAX_ENTRIES: 256, + METADATA_KEY_MAX: 128, + METADATA_VALUE_MAX: 1024, + LABELS_MAX: 64, + LABEL_LENGTH_MAX: 512, + RELATED_RESOURCES_MAX: 100, + DATAPOINT_VALUE_MAX: 64 +}); + +window.LimitErrors = (function () { + + const TYPE = { + RATE: 'https://intellistream.ai/errors/rate-limit-exceeded', + QUOTA: 'https://intellistream.ai/errors/ingest-quota-exceeded', + TENANT: 'https://intellistream.ai/errors/tenant-limit-reached', + TOO_LARGE: 'https://intellistream.ai/errors/request-too-large' + }; + + /** "in about 2 minutes" reads better than "in 118 seconds", and the number is approximate anyway. */ + function waitFor(seconds) { + const s = Number(seconds); + if (!Number.isFinite(s) || s <= 0) return $L('error.limit.wait.moment'); + if (s < 90) return $L('error.limit.wait.seconds', null, [Math.ceil(s)]); + if (s < 5400) return $L('error.limit.wait.minutes', null, [Math.round(s / 60)]); + return $L('error.limit.wait.hours', null, [Math.round(s / 3600)]); + } + + function bytes(n) { + const b = Number(n); + if (!Number.isFinite(b) || b <= 0) return null; + return window.getByteSize ? window.getByteSize(b) : b + ' B'; + } + + /** + * The api names the thing it counted in English, for whoever reads a log. Translate the ones we + * know and fall back to the server's own word for anything else, so a metric added later still + * produces a sentence rather than a gap. + */ + const METRIC_KEYS = { + 'events': 'error.limit.metric.events', + 'nodes': 'error.limit.metric.nodes', + 'relationships': 'error.limit.metric.relationships', + 'data points': 'error.limit.metric.datapoints', + 'text data points': 'error.limit.metric.text.datapoints', + 'ingested bytes': 'error.limit.metric.bytes' + }; + + function metricName(metric) { + if (!metric) return null; + const key = METRIC_KEYS[metric]; + return key ? $L(key) : metric; + } + + function isLimit(json) { + return !!(json && typeof json === 'object' && json.type + && Object.values(TYPE).indexOf(json.type) !== -1); + } + + /** The localized message for a problem document, or null if it is not one of ours. */ + function message(json) { + if (!isLimit(json)) return null; + switch (json.type) { + case TYPE.RATE: + return $L('error.limit.rate', null, [waitFor(json.retryAfter)]); + case TYPE.QUOTA: + return $L('error.limit.quota', null, [waitFor(json.retryAfter)]); + case TYPE.TENANT: { + const what = metricName(json.metric); + return what + ? $L('error.limit.tenant', null, [json.limit, what]) + : $L('error.limit.tenant.plain'); + } + case TYPE.TOO_LARGE: { + const max = bytes(json.limitBytes); + return max ? $L('error.limit.too.large', null, [max]) : $L('error.limit.too.large.plain'); + } + default: + return null; + } + } + + /** + * A message from a status code alone, for the places that never parse the body: the raw-body file + * upload reads `xhr.responseText`, which without this shows the caller a JSON document. + * + * `body` is the raw text, parsed here when it happens to be a problem document so the numbers in + * it are still used; otherwise the status carries the meaning on its own. + */ + function fromStatus(status, body) { + if (body) { + try { + const parsed = JSON.parse(body); + const parsedMessage = message(parsed); + if (parsedMessage) return parsedMessage; + } catch (ignored) { + // Not JSON, so there is nothing in it to read. The status below still says enough. + } + } + if (status === 429) return $L('error.limit.rate.plain'); + if (status === 413) return $L('error.limit.too.large.plain'); + if (status === 403) return $L('error.limit.tenant.plain'); + return null; + } + + return { TYPE: TYPE, isLimit: isLimit, message: message, fromStatus: fromStatus }; +})(); diff --git a/datahub-console/src/main/resources/static/js/right-form-content/base_form_abstract.js b/datahub-console/src/main/resources/static/js/right-form-content/base_form_abstract.js index 0e689f7b..61309930 100644 --- a/datahub-console/src/main/resources/static/js/right-form-content/base_form_abstract.js +++ b/datahub-console/src/main/resources/static/js/right-form-content/base_form_abstract.js @@ -492,6 +492,8 @@ class DatasetFormAbstract extends BaseFormAbstract{ if(json.type === 'https://intellistream.ai/errors/permissions-unavailable'){ return $L('error.permissions.unavailable'); } + const limit = window.LimitErrors && window.LimitErrors.message(json); + if(limit) return limit; return json.detail || json.title; } @@ -785,8 +787,10 @@ class DatasetFormAbstract extends BaseFormAbstract{ const keyCell = row.insertCell(-1); const valueCell = row.insertCell(-1); const btnCell = row.insertCell(-1); - keyCell.innerHTML = ``; - valueCell.innerHTML = ``; + // maxlength mirrors the api's own caps (FieldLimits), so an over-long key or value is stopped + // where it is typed rather than after a round trip that refuses the whole form. + keyCell.innerHTML = ``; + valueCell.innerHTML = ``; const trashBtn = Object.assign(document.createElement('button'), { className: "dh-btn secondary small", innerHTML: `` diff --git a/datahub-console/src/main/resources/static/js/right-form-content/datasets/form.js b/datahub-console/src/main/resources/static/js/right-form-content/datasets/form.js index 8e7d8acb..7bf3cc1a 100644 --- a/datahub-console/src/main/resources/static/js/right-form-content/datasets/form.js +++ b/datahub-console/src/main/resources/static/js/right-form-content/datasets/form.js @@ -54,7 +54,7 @@ class DataSetForm extends DatasetFormAbstract {

${$L('external.id.charset.help')}

- + `; } diff --git a/datahub-console/src/main/resources/static/js/right-form-content/files/form.js b/datahub-console/src/main/resources/static/js/right-form-content/files/form.js index e2e7333d..ad1ad2c0 100644 --- a/datahub-console/src/main/resources/static/js/right-form-content/files/form.js +++ b/datahub-console/src/main/resources/static/js/right-form-content/files/form.js @@ -193,7 +193,7 @@ class UploadFileForm extends FileFormAbstract { placeholder="${$L('write.external.id.here')}" tabindex="50"/> - + @@ -417,7 +417,10 @@ class UploadFileForm extends FileFormAbstract { document.location = '/files/list'; } } else { - this.handleUploadError(xhr.responseText || $L('upload.failed')); + // The raw body is a problem document, so show what it means rather than its JSON. + this.handleUploadError( + window.LimitErrors.fromStatus(xhr.status, xhr.responseText) + || $L('upload.failed')); } }); @@ -699,7 +702,8 @@ class SetRelatedResourcesForm extends FileFormAbstract { Flash.info($L('file.updated')); } else { this.submitButtonElement.disabled = false; - response.text().then(t => Flash.error(t || $L('update.failed'))); + response.text().then(t => + Flash.error(window.LimitErrors.fromStatus(response.status, t) || t || $L('update.failed'))); } }) .catch(() => { @@ -732,7 +736,7 @@ class UpdateFileForm extends FileFormAbstract { - + @@ -801,7 +805,8 @@ class UpdateFileForm extends FileFormAbstract { document.location.reload(); } else { this.submitButtonElement.disabled = false; - response.text().then(t => Flash.error(t || $L('update.failed'))); + response.text().then(t => + Flash.error(window.LimitErrors.fromStatus(response.status, t) || t || $L('update.failed'))); } }) .catch(() => { diff --git a/datahub-console/src/main/resources/static/js/right-form-content/resources/form.js b/datahub-console/src/main/resources/static/js/right-form-content/resources/form.js index cb15e779..e01e3fd1 100644 --- a/datahub-console/src/main/resources/static/js/right-form-content/resources/form.js +++ b/datahub-console/src/main/resources/static/js/right-form-content/resources/form.js @@ -93,7 +93,7 @@ class ResourceForm extends DatasetFormAbstract{

${$L('external.id.charset.help')}

- + ${$L('name')} - + @@ -901,7 +901,7 @@ class RelationForm extends DatasetFormAbstract{ type="text" name="i18nCode" value="${this.getFormProperty('i18nCode')}" tabindex="20" /> - + `; } @@ -1142,7 +1142,7 @@ class ResourceEdgeForm extends DatasetFormAbstract{ - +