diff --git a/design/mnemosyne/mnemosyne-s1-deployable-per-shard.excalidraw b/design/mnemosyne/mnemosyne-s1-deployable-per-shard.excalidraw new file mode 100644 index 00000000..595ed658 --- /dev/null +++ b/design/mnemosyne/mnemosyne-s1-deployable-per-shard.excalidraw @@ -0,0 +1,1671 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "id": "el47", + "x": 40, + "y": -10, + "width": 700, + "height": 27.5, + "angle": 0, + "strokeColor": "#0b7285", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 390452953, + "version": 1, + "versionNonce": 202363286, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "Mnemosyne - S1: Deployable-per-shard (read-replicas, Envoy LB)", + "originalText": "Mnemosyne - S1: Deployable-per-shard (read-replicas, Envoy LB)", + "fontSize": 22, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 19, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el1", + "x": 60, + "y": 60, + "width": 210, + "height": 70, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 478163328, + "version": 1, + "versionNonce": 107420370, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el2", + "x": 60, + "y": 78.75, + "width": 210, + "height": 32.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1181241944, + "version": 1, + "versionNonce": 1051802513, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "Databricks / PySpark Job\nshard + version -> files", + "originalText": "Databricks / PySpark Job\nshard + version -> files", + "fontSize": 13, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 11, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el3", + "x": 330, + "y": 60, + "width": 170, + "height": 70, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 958682847, + "version": 1, + "versionNonce": 599310826, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el4", + "x": 330, + "y": 70.625, + "width": 170, + "height": 48.75, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 440213416, + "version": 1, + "versionNonce": 373399427, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "GCS\nversioned files\n(date + run)", + "originalText": "GCS\nversioned files\n(date + run)", + "fontSize": 13, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 11, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el5", + "x": 760, + "y": 50, + "width": 310, + "height": 95, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1812140442, + "version": 1, + "versionNonce": 136505588, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el6", + "x": 760, + "y": 73.125, + "width": 310, + "height": 48.75, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 127978095, + "version": 1, + "versionNonce": 402418011, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "CONTROL PLANE\nonboarding . sizing . cost\nversion pointer (etcd)", + "originalText": "CONTROL PLANE\nonboarding . sizing . cost\nversion pointer (etcd)", + "fontSize": 13, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 11, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el7", + "x": 1140, + "y": 55, + "width": 250, + "height": 90, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#99e9f2", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 939042956, + "version": 1, + "versionNonce": 999270937, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el8", + "x": 1140, + "y": 75.625, + "width": 250, + "height": 48.75, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 113971124, + "version": 1, + "versionNonce": 854001194, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "CLIENT SDK\nkey -> shard -> endpoint\nscatter-gather + combine", + "originalText": "CLIENT SDK\nkey -> shard -> endpoint\nscatter-gather + combine", + "fontSize": 13, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 11, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el9", + "x": 1140, + "y": 170, + "width": 250, + "height": 48, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffd43b", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1801823909, + "version": 1, + "versionNonce": 946785249, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el10", + "x": 1140, + "y": 185.875, + "width": 250, + "height": 16.25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1929338155, + "version": 1, + "versionNonce": 1194819985, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "Feature Store (tenant)", + "originalText": "Feature Store (tenant)", + "fontSize": 13, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 11, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el11", + "x": 270, + "y": 95, + "width": 60, + "height": 0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 27911968, + "version": 1, + "versionNonce": 685731525, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 60, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el12", + "x": 500, + "y": 95, + "width": 260, + "height": 2, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1815115026, + "version": 1, + "versionNonce": 1461364855, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 260, + 2 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el13", + "x": 1265, + "y": 170, + "width": 0, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1193448330, + "version": 1, + "versionNonce": 667779377, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 0, + -25 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el14", + "x": 1070, + "y": 90, + "width": 70, + "height": 5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 924765564, + "version": 1, + "versionNonce": 1445662586, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 70, + 5 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el15", + "x": 1015, + "y": 30, + "width": 300, + "height": 13.75, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 438989806, + "version": 1, + "versionNonce": 398340370, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "watch topology + version", + "originalText": "watch topology + version", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el16", + "x": 120, + "y": 250, + "width": 330, + "height": 270, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#f1f3f5", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1631775358, + "version": 1, + "versionNonce": 415393688, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el17", + "x": 132, + "y": 258, + "width": 300, + "height": 17.5, + "angle": 0, + "strokeColor": "#212529", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1541804687, + "version": 1, + "versionNonce": 1477278578, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "Shard 1 (deployable)", + "originalText": "Shard 1 (deployable)", + "fontSize": 14, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 12, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el18", + "x": 150, + "y": 300, + "width": 270, + "height": 40, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#eebefa", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1136108455, + "version": 1, + "versionNonce": 186618212, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el19", + "x": 150, + "y": 312.5, + "width": 270, + "height": 15.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1973214823, + "version": 1, + "versionNonce": 536124281, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "Envoy / Service (intra-shard LB)", + "originalText": "Envoy / Service (intra-shard LB)", + "fontSize": 12, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 10, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el20", + "x": 150, + "y": 360, + "width": 125, + "height": 140, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1625792788, + "version": 1, + "versionNonce": 338444265, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el21", + "x": 150, + "y": 395.625, + "width": 125, + "height": 68.75, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1259191106, + "version": 1, + "versionNonce": 1553210609, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "pod A\n--------\ndata-loader\nengine (RocksDB)\nread server (TCP)", + "originalText": "pod A\n--------\ndata-loader\nengine (RocksDB)\nread server (TCP)", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el22", + "x": 295, + "y": 360, + "width": 125, + "height": 140, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 825873197, + "version": 1, + "versionNonce": 298737107, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el23", + "x": 295, + "y": 395.625, + "width": 125, + "height": 68.75, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 196814234, + "version": 1, + "versionNonce": 978815631, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "pod B\n--------\ndata-loader\nengine (RocksDB)\nread server (TCP)", + "originalText": "pod B\n--------\ndata-loader\nengine (RocksDB)\nread server (TCP)", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el24", + "x": 215, + "y": 340, + "width": 3, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1242911822, + "version": 1, + "versionNonce": 342703922, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -3, + 20 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el25", + "x": 355, + "y": 340, + "width": 2, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 999829241, + "version": 1, + "versionNonce": 433797841, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 2, + 20 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el26", + "x": 490, + "y": 250, + "width": 330, + "height": 270, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#f1f3f5", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1632629720, + "version": 1, + "versionNonce": 1193887546, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el27", + "x": 502, + "y": 258, + "width": 300, + "height": 17.5, + "angle": 0, + "strokeColor": "#212529", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1947382420, + "version": 1, + "versionNonce": 1566942274, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "Shard 2 (deployable)", + "originalText": "Shard 2 (deployable)", + "fontSize": 14, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 12, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el28", + "x": 520, + "y": 300, + "width": 270, + "height": 40, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#eebefa", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 698594026, + "version": 1, + "versionNonce": 1589915145, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el29", + "x": 520, + "y": 312.5, + "width": 270, + "height": 15.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1525876052, + "version": 1, + "versionNonce": 899825839, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "Envoy / Service (intra-shard LB)", + "originalText": "Envoy / Service (intra-shard LB)", + "fontSize": 12, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 10, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el30", + "x": 520, + "y": 360, + "width": 125, + "height": 140, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1146660998, + "version": 1, + "versionNonce": 306671448, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el31", + "x": 520, + "y": 395.625, + "width": 125, + "height": 68.75, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 735034882, + "version": 1, + "versionNonce": 1051454924, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "pod C\n--------\ndata-loader\nengine (RocksDB)\nread server (TCP)", + "originalText": "pod C\n--------\ndata-loader\nengine (RocksDB)\nread server (TCP)", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el32", + "x": 665, + "y": 360, + "width": 125, + "height": 140, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 701808368, + "version": 1, + "versionNonce": 1985392499, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el33", + "x": 665, + "y": 395.625, + "width": 125, + "height": 68.75, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1629748728, + "version": 1, + "versionNonce": 1159417076, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "pod D\n--------\ndata-loader\nengine (RocksDB)\nread server (TCP)", + "originalText": "pod D\n--------\ndata-loader\nengine (RocksDB)\nread server (TCP)", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el34", + "x": 585, + "y": 340, + "width": 3, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 943239975, + "version": 1, + "versionNonce": 1392783744, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -3, + 20 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el35", + "x": 725, + "y": 340, + "width": 2, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 240251662, + "version": 1, + "versionNonce": 983753978, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 2, + 20 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el36", + "x": 860, + "y": 250, + "width": 200, + "height": 270, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#f1f3f5", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 137869476, + "version": 1, + "versionNonce": 1354860541, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el37", + "x": 860, + "y": 360, + "width": 200, + "height": 45.0, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1722989660, + "version": 1, + "versionNonce": 1149938335, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "Shard N\n. . .", + "originalText": "Shard N\n. . .", + "fontSize": 18, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 16, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el38", + "x": 1180, + "y": 145, + "width": 710, + "height": 155, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 284277890, + "version": 1, + "versionNonce": 906164397, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -710, + 155 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el39", + "x": 640, + "y": 150, + "width": 300, + "height": 13.75, + "angle": 0, + "strokeColor": "#1971c2", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1351531224, + "version": 1, + "versionNonce": 913224048, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "route: key % N -> shard", + "originalText": "route: key % N -> shard", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el40", + "x": 1160, + "y": 150, + "width": 340, + "height": 150, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 2144181938, + "version": 1, + "versionNonce": 1699226065, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -340, + 150 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el41", + "x": 415, + "y": 130, + "width": 115, + "height": 170, + "angle": 0, + "strokeColor": "#2f9e44", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1970753706, + "version": 1, + "versionNonce": 613628804, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -115, + 170 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el42", + "x": 300, + "y": 190, + "width": 220, + "height": 12.5, + "angle": 0, + "strokeColor": "#2f9e44", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1137651679, + "version": 1, + "versionNonce": 599707678, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "data-loader pulls files", + "originalText": "data-loader pulls files", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el43", + "x": 840, + "y": 140, + "width": 370, + "height": 155, + "angle": 0, + "strokeColor": "#e8590c", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1059257081, + "version": 1, + "versionNonce": 1128466621, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -370, + 155 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el44", + "x": 560, + "y": 210, + "width": 260, + "height": 12.5, + "angle": 0, + "strokeColor": "#e8590c", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1840109256, + "version": 1, + "versionNonce": 1715412120, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "assign shard + version", + "originalText": "assign shard + version", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el45", + "x": 120, + "y": 560, + "width": 940, + "height": 46, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff9db", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1554762904, + "version": 1, + "versionNonce": 941975481, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el46", + "x": 120, + "y": 575.5, + "width": 940, + "height": 15.0, + "angle": 0, + "strokeColor": "#5c3d00", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 594130309, + "version": 1, + "versionNonce": 2119634400, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059945, + "link": null, + "locked": false, + "type": "text", + "text": "S1: static shard->deployable . per-shard read-replicas (masterless) . Envoy intra-shard LB . scale replicas per shard", + "originalText": "S1: static shard->deployable . per-shard read-replicas (masterless) . Envoy intra-shard LB . scale replicas per shard", + "fontSize": 12, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 10, + "containerId": null, + "lineHeight": 1.25 + } + ], + "appState": { + "viewBackgroundColor": "#ffffff", + "gridSize": null + }, + "files": {} +} \ No newline at end of file diff --git a/design/mnemosyne/mnemosyne-s2-scattered-regions.excalidraw b/design/mnemosyne/mnemosyne-s2-scattered-regions.excalidraw new file mode 100644 index 00000000..0e767d91 --- /dev/null +++ b/design/mnemosyne/mnemosyne-s2-scattered-regions.excalidraw @@ -0,0 +1,1382 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "id": "el85", + "x": 40, + "y": -10, + "width": 700, + "height": 27.5, + "angle": 0, + "strokeColor": "#0b7285", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 568275056, + "version": 1, + "versionNonce": 2030106618, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "text", + "text": "Mnemosyne - S2: Scattered regions (TiDB-like, PD-managed)", + "originalText": "Mnemosyne - S2: Scattered regions (TiDB-like, PD-managed)", + "fontSize": 22, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 19, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el48", + "x": 60, + "y": 60, + "width": 210, + "height": 70, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 470939446, + "version": 1, + "versionNonce": 656448480, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el49", + "x": 60, + "y": 78.75, + "width": 210, + "height": 32.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 687117442, + "version": 1, + "versionNonce": 1813163241, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "text", + "text": "Databricks / PySpark Job\nregion + version -> files", + "originalText": "Databricks / PySpark Job\nregion + version -> files", + "fontSize": 13, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 11, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el50", + "x": 330, + "y": 60, + "width": 170, + "height": 70, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 272849425, + "version": 1, + "versionNonce": 1652563014, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el51", + "x": 330, + "y": 70.625, + "width": 170, + "height": 48.75, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1639042339, + "version": 1, + "versionNonce": 2010258947, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "text", + "text": "GCS\nversioned files\n(date + run)", + "originalText": "GCS\nversioned files\n(date + run)", + "fontSize": 13, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 11, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el52", + "x": 690, + "y": 45, + "width": 340, + "height": 105, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1079815405, + "version": 1, + "versionNonce": 49310609, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el53", + "x": 690, + "y": 75.0, + "width": 340, + "height": 45.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 491995980, + "version": 1, + "versionNonce": 1146005449, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "text", + "text": "CONTROL PLANE - PLACEMENT DRIVER\nregion catalog . rebalancer\nheartbeats . version ptr (etcd)", + "originalText": "CONTROL PLANE - PLACEMENT DRIVER\nregion catalog . rebalancer\nheartbeats . version ptr (etcd)", + "fontSize": 12, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 10, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el54", + "x": 1090, + "y": 55, + "width": 260, + "height": 95, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#99e9f2", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1461042544, + "version": 1, + "versionNonce": 479112938, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el55", + "x": 1090, + "y": 80.0, + "width": 260, + "height": 45.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1260573449, + "version": 1, + "versionNonce": 1867302555, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "text", + "text": "CLIENT SDK\nkey -> region -> pod (1 of RF)\nregion-map + 'moved' redirects", + "originalText": "CLIENT SDK\nkey -> region -> pod (1 of RF)\nregion-map + 'moved' redirects", + "fontSize": 12, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 10, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el56", + "x": 1090, + "y": 175, + "width": 260, + "height": 48, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffd43b", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 679282379, + "version": 1, + "versionNonce": 1948728484, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el57", + "x": 1090, + "y": 190.875, + "width": 260, + "height": 16.25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 13938522, + "version": 1, + "versionNonce": 1131247331, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "text", + "text": "Feature Store (tenant)", + "originalText": "Feature Store (tenant)", + "fontSize": 13, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 11, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el58", + "x": 270, + "y": 95, + "width": 60, + "height": 0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 767303989, + "version": 1, + "versionNonce": 457031005, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 60, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el59", + "x": 500, + "y": 95, + "width": 190, + "height": 5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1281810601, + "version": 1, + "versionNonce": 854316682, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 190, + 5 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el60", + "x": 1220, + "y": 175, + "width": 0, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 656439678, + "version": 1, + "versionNonce": 1605947757, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 0, + -25 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el61", + "x": 1030, + "y": 95, + "width": 60, + "height": 5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 693847830, + "version": 1, + "versionNonce": 2456274, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 60, + 5 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el62", + "x": 975, + "y": 28, + "width": 320, + "height": 13.75, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1392239676, + "version": 1, + "versionNonce": 2098545542, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "text", + "text": "watch region-map + version", + "originalText": "watch region-map + version", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el63", + "x": 110, + "y": 310, + "width": 185, + "height": 165, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffe3e3", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 83651971, + "version": 1, + "versionNonce": 480468220, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el64", + "x": 110, + "y": 362.5, + "width": 185, + "height": 60.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1558990518, + "version": 1, + "versionNonce": 1320763136, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "text", + "text": "Pod 1\n[loader + engine + server]\n--------\nregions: r1, r4, r9", + "originalText": "Pod 1\n[loader + engine + server]\n--------\nregions: r1, r4, r9", + "fontSize": 12, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 10, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el65", + "x": 330, + "y": 310, + "width": 185, + "height": 165, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffe3e3", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1028439864, + "version": 1, + "versionNonce": 248786715, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el66", + "x": 330, + "y": 362.5, + "width": 185, + "height": 60.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1034535594, + "version": 1, + "versionNonce": 338258952, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "text", + "text": "Pod 2\n[loader + engine + server]\n--------\nregions: r2, r4, r7", + "originalText": "Pod 2\n[loader + engine + server]\n--------\nregions: r2, r4, r7", + "fontSize": 12, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 10, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el67", + "x": 550, + "y": 310, + "width": 185, + "height": 165, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffe3e3", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 367878762, + "version": 1, + "versionNonce": 2087313122, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el68", + "x": 550, + "y": 362.5, + "width": 185, + "height": 60.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 297265481, + "version": 1, + "versionNonce": 540137297, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "text", + "text": "Pod 3\n[loader + engine + server]\n--------\nregions: r3, r7, r9", + "originalText": "Pod 3\n[loader + engine + server]\n--------\nregions: r3, r7, r9", + "fontSize": 12, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 10, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el69", + "x": 770, + "y": 310, + "width": 185, + "height": 165, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffe3e3", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 551437150, + "version": 1, + "versionNonce": 2041322271, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el70", + "x": 770, + "y": 362.5, + "width": 185, + "height": 60.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 709214892, + "version": 1, + "versionNonce": 1138409540, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "text", + "text": "Pod 4\n[loader + engine + server]\n--------\nregions: r1, r2, r3", + "originalText": "Pod 4\n[loader + engine + server]\n--------\nregions: r1, r2, r3", + "fontSize": 12, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 10, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el71", + "x": 295, + "y": 392, + "width": 35, + "height": 0, + "angle": 0, + "strokeColor": "#9c36b5", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1817363616, + "version": 1, + "versionNonce": 909666338, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 35, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el72", + "x": 330, + "y": 400, + "width": 35, + "height": 0, + "angle": 0, + "strokeColor": "#9c36b5", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 863937248, + "version": 1, + "versionNonce": 1338811291, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -35, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el73", + "x": 735, + "y": 392, + "width": 35, + "height": 0, + "angle": 0, + "strokeColor": "#9c36b5", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1713658875, + "version": 1, + "versionNonce": 1603828850, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 35, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el74", + "x": 770, + "y": 400, + "width": 35, + "height": 0, + "angle": 0, + "strokeColor": "#9c36b5", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1881625506, + "version": 1, + "versionNonce": 1939118224, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -35, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el75", + "x": 300, + "y": 470, + "width": 520, + "height": 13.75, + "angle": 0, + "strokeColor": "#9c36b5", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 519709080, + "version": 1, + "versionNonce": 1064746526, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "text", + "text": "dashed = region transfer / rebalance (RF=2: each region on 2 pods)", + "originalText": "dashed = region transfer / rebalance (RF=2: each region on 2 pods)", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el76", + "x": 820, + "y": 150, + "width": 390, + "height": 160, + "angle": 0, + "strokeColor": "#e8590c", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 965067728, + "version": 1, + "versionNonce": 274989149, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -390, + 160 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el77", + "x": 540, + "y": 200, + "width": 280, + "height": 12.5, + "angle": 0, + "strokeColor": "#e8590c", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1452066460, + "version": 1, + "versionNonce": 90341466, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "text", + "text": "assign regions + heartbeat", + "originalText": "assign regions + heartbeat", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el78", + "x": 1140, + "y": 150, + "width": 500, + "height": 160, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 988335252, + "version": 1, + "versionNonce": 945826487, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -500, + 160 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el79", + "x": 720, + "y": 165, + "width": 320, + "height": 13.75, + "angle": 0, + "strokeColor": "#1971c2", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 30884439, + "version": 1, + "versionNonce": 304912983, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "text", + "text": "route by region", + "originalText": "route by region", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el80", + "x": 1120, + "y": 155, + "width": 260, + "height": 155, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 252860897, + "version": 1, + "versionNonce": 983297493, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -260, + 155 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el81", + "x": 415, + "y": 130, + "width": 215, + "height": 180, + "angle": 0, + "strokeColor": "#2f9e44", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 289482183, + "version": 1, + "versionNonce": 134917628, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -215, + 180 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el82", + "x": 210, + "y": 200, + "width": 220, + "height": 12.5, + "angle": 0, + "strokeColor": "#2f9e44", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1419179581, + "version": 1, + "versionNonce": 304330004, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "text", + "text": "data-loader pulls regions", + "originalText": "data-loader pulls regions", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el83", + "x": 110, + "y": 560, + "width": 845, + "height": 46, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff9db", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1022222126, + "version": 1, + "versionNonce": 1196050748, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el84", + "x": 110, + "y": 575.5, + "width": 845, + "height": 15.0, + "angle": 0, + "strokeColor": "#5c3d00", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 2084839400, + "version": 1, + "versionNonce": 920140091, + "isDeleted": false, + "boundElements": [], + "updated": 1779905059947, + "link": null, + "locked": false, + "type": "text", + "text": "S2: dynamic region->pod placement . PD-managed rebalancing . cluster-level scaling . fungible/packed capacity", + "originalText": "S2: dynamic region->pod placement . PD-managed rebalancing . cluster-level scaling . fungible/packed capacity", + "fontSize": 12, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 10, + "containerId": null, + "lineHeight": 1.25 + } + ], + "appState": { + "viewBackgroundColor": "#ffffff", + "gridSize": null + }, + "files": {} +} \ No newline at end of file diff --git a/mnemosyne-platform-architecture.excalidraw b/mnemosyne-platform-architecture.excalidraw new file mode 100644 index 00000000..cc9e6167 --- /dev/null +++ b/mnemosyne-platform-architecture.excalidraw @@ -0,0 +1,4873 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "id": "el142", + "x": 40, + "y": -28, + "width": 1300, + "height": 27.5, + "angle": 0, + "strokeColor": "#0b7285", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 874443788, + "version": 1, + "versionNonce": 1133802215, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "Mnemosyne - Platform Architecture (onboarding . multi-tenant . internal scaling)", + "originalText": "Mnemosyne - Platform Architecture (onboarding . multi-tenant . internal scaling)", + "fontSize": 22, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 19, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el1", + "x": 60, + "y": 30, + "width": 180, + "height": 80, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 478163328, + "version": 1, + "versionNonce": 107420370, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el2", + "x": 60, + "y": 55.0, + "width": 180, + "height": 30.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1181241944, + "version": 1, + "versionNonce": 1051802513, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Tenant Admin\n(onboarding team)", + "originalText": "Tenant Admin\n(onboarding team)", + "fontSize": 12, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 10, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el3", + "x": 240, + "y": 70, + "width": 60, + "height": 0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 958682847, + "version": 1, + "versionNonce": 599310826, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 60, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el4", + "x": 300, + "y": 30, + "width": 300, + "height": 100, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 440213416, + "version": 1, + "versionNonce": 373399427, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el5", + "x": 300, + "y": 59.375, + "width": 300, + "height": 41.25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1812140442, + "version": 1, + "versionNonce": 136505588, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "ONBOARDING API\ninputs: D, target RPS, p99,\nrollback SLA, $-target", + "originalText": "ONBOARDING API\ninputs: D, target RPS, p99,\nrollback SLA, $-target", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el6", + "x": 600, + "y": 80, + "width": 60, + "height": 0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 127978095, + "version": 1, + "versionNonce": 402418011, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 60, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el7", + "x": 660, + "y": 30, + "width": 300, + "height": 100, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 939042956, + "version": 1, + "versionNonce": 999270937, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el8", + "x": 660, + "y": 59.375, + "width": 300, + "height": 41.25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 113971124, + "version": 1, + "versionNonce": 854001194, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "SIZER -> outputs\n{S, R, V, pod-spec,\nautoscaling policy, cost}", + "originalText": "SIZER -> outputs\n{S, R, V, pod-spec,\nautoscaling policy, cost}", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el9", + "x": 960, + "y": 80, + "width": 60, + "height": 0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1801823909, + "version": 1, + "versionNonce": 946785249, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 60, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el10", + "x": 1020, + "y": 30, + "width": 360, + "height": 100, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1929338155, + "version": 1, + "versionNonce": 1194819985, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el11", + "x": 1020, + "y": 59.375, + "width": 360, + "height": 41.25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 27911968, + "version": 1, + "versionNonce": 685731525, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Returned to tenant:\n* Topology endpoint\n* SDK snippet * onboarding docs", + "originalText": "Returned to tenant:\n* Topology endpoint\n* SDK snippet * onboarding docs", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el12", + "x": 60, + "y": 170, + "width": 240, + "height": 90, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1815115026, + "version": 1, + "versionNonce": 1461364855, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el13", + "x": 60, + "y": 194.375, + "width": 240, + "height": 41.25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1193448330, + "version": 1, + "versionNonce": 667779377, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Databricks / PySpark Job\nshard + version -> files\n(cron-scheduled per tenant)", + "originalText": "Databricks / PySpark Job\nshard + version -> files\n(cron-scheduled per tenant)", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el14", + "x": 300, + "y": 215, + "width": 60, + "height": 0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 924765564, + "version": 1, + "versionNonce": 1445662586, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 60, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el15", + "x": 360, + "y": 170, + "width": 300, + "height": 90, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 438989806, + "version": 1, + "versionNonce": 398340370, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el16", + "x": 360, + "y": 194.375, + "width": 300, + "height": 41.25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1631775358, + "version": 1, + "versionNonce": 415393688, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "GCS (shared)\nversioned per-tenant files\ngs://.../{tenant}/{store}/{vId}/...", + "originalText": "GCS (shared)\nversioned per-tenant files\ngs://.../{tenant}/{store}/{vId}/...", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el17", + "x": 60, + "y": 290, + "width": 1400, + "height": 180, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1541804687, + "version": 1, + "versionNonce": 1477278578, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el18", + "x": 80, + "y": 300, + "width": 1360, + "height": 18.75, + "angle": 0, + "strokeColor": "#0b4f6c", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1136108455, + "version": 1, + "versionNonce": 186618212, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "CONTROL PLANE . etcd-backed . single-writer activeVersion (CAS on topologyVersion)", + "originalText": "CONTROL PLANE . etcd-backed . single-writer activeVersion (CAS on topologyVersion)", + "fontSize": 15, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 13, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el19", + "x": 80, + "y": 340, + "width": 240, + "height": 120, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1973214823, + "version": 1, + "versionNonce": 536124281, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el20", + "x": 80, + "y": 379.375, + "width": 240, + "height": 41.25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1625792788, + "version": 1, + "versionNonce": 338444265, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "PLACEMENT AUTHORITY\nshard -> pod assignment\n(consistent hashing)", + "originalText": "PLACEMENT AUTHORITY\nshard -> pod assignment\n(consistent hashing)", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el21", + "x": 340, + "y": 340, + "width": 240, + "height": 120, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1259191106, + "version": 1, + "versionNonce": 1553210609, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el22", + "x": 340, + "y": 379.375, + "width": 240, + "height": 41.25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 825873197, + "version": 1, + "versionNonce": 298737107, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "VERSION LIFECYCLE\nbuild -> ready -> active\n-> retire . rollback", + "originalText": "VERSION LIFECYCLE\nbuild -> ready -> active\n-> retire . rollback", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el23", + "x": 600, + "y": 340, + "width": 240, + "height": 120, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 196814234, + "version": 1, + "versionNonce": 978815631, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el24", + "x": 600, + "y": 379.375, + "width": 240, + "height": 41.25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1242911822, + "version": 1, + "versionNonce": 342703922, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "KEDA POLICY MGR\nstorage-bound: pinned\ncompute-bound: RPS+p99", + "originalText": "KEDA POLICY MGR\nstorage-bound: pinned\ncompute-bound: RPS+p99", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el25", + "x": 860, + "y": 340, + "width": 240, + "height": 120, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 999829241, + "version": 1, + "versionNonce": 433797841, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el26", + "x": 860, + "y": 379.375, + "width": 240, + "height": 41.25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1632629720, + "version": 1, + "versionNonce": 1193887546, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "WARM-COVERAGE GATE\nflip only when all shards\nreport version warm", + "originalText": "WARM-COVERAGE GATE\nflip only when all shards\nreport version warm", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el27", + "x": 1120, + "y": 340, + "width": 320, + "height": 120, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff9db", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1947382420, + "version": 1, + "versionNonce": 1566942274, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el28", + "x": 1120, + "y": 368.75, + "width": 320, + "height": 62.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 698594026, + "version": 1, + "versionNonce": 1589915145, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "ETCD STATE\n/tenants/{t}/stores/{s}/\n activeVersion . rollbackVersion\n versions/{vId} . assignment\n scaling/policy", + "originalText": "ETCD STATE\n/tenants/{t}/stores/{s}/\n activeVersion . rollbackVersion\n versions/{vId} . assignment\n scaling/policy", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el29", + "x": 1200, + "y": 130, + "width": 0, + "height": 160, + "angle": 0, + "strokeColor": "#e8590c", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1525876052, + "version": 1, + "versionNonce": 899825839, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 0, + 160 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el30", + "x": 1010, + "y": 145, + "width": 400, + "height": 12.5, + "angle": 0, + "strokeColor": "#e8590c", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1146660998, + "version": 1, + "versionNonce": 306671448, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "writes tenant config + cron schedule", + "originalText": "writes tenant config + cron schedule", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el31", + "x": 560, + "y": 260, + "width": 160, + "height": 30, + "angle": 0, + "strokeColor": "#2f9e44", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 735034882, + "version": 1, + "versionNonce": 1051454924, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 160, + 30 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el32", + "x": 480, + "y": 265, + "width": 260, + "height": 12.5, + "angle": 0, + "strokeColor": "#2f9e44", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 701808368, + "version": 1, + "versionNonce": 1985392499, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "version manifest", + "originalText": "version manifest", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el33", + "x": 60, + "y": 490, + "width": 1400, + "height": 230, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#f8f9fa", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1629748728, + "version": 1, + "versionNonce": 1159417076, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el34", + "x": 80, + "y": 498, + "width": 1360, + "height": 16.25, + "angle": 0, + "strokeColor": "#212529", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 943239975, + "version": 1, + "versionNonce": 1392783744, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Tenant A . Feature Store (10 TB / 50k peak - storage-bound - S=10, R=2, V=2 - showing 3 of 10 shards)", + "originalText": "Tenant A . Feature Store (10 TB / 50k peak - storage-bound - S=10, R=2, V=2 - showing 3 of 10 shards)", + "fontSize": 13, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 11, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el35", + "x": 80, + "y": 528, + "width": 160, + "height": 60, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffd43b", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 240251662, + "version": 1, + "versionNonce": 983753978, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el36", + "x": 80, + "y": 550.5, + "width": 160, + "height": 15.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 137869476, + "version": 1, + "versionNonce": 1354860541, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Feature Store", + "originalText": "Feature Store", + "fontSize": 12, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 10, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el37", + "x": 260, + "y": 528, + "width": 200, + "height": 60, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#99e9f2", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1722989660, + "version": 1, + "versionNonce": 1149938335, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el38", + "x": 260, + "y": 544.25, + "width": 200, + "height": 27.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 284277890, + "version": 1, + "versionNonce": 906164397, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Client SDK\nkey -> shard -> pod", + "originalText": "Client SDK\nkey -> shard -> pod", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el39", + "x": 240, + "y": 558, + "width": 20, + "height": 0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1351531224, + "version": 1, + "versionNonce": 913224048, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 20, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el40", + "x": 500, + "y": 528, + "width": 290, + "height": 180, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 2144181938, + "version": 1, + "versionNonce": 1699226065, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el41", + "x": 510, + "y": 536, + "width": 270, + "height": 13.75, + "angle": 0, + "strokeColor": "#212529", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1970753706, + "version": 1, + "versionNonce": 613628804, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Shard 1 deployable", + "originalText": "Shard 1 deployable", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el42", + "x": 510, + "y": 560, + "width": 270, + "height": 28, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#eebefa", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1137651679, + "version": 1, + "versionNonce": 599707678, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el43", + "x": 510, + "y": 567.75, + "width": 270, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1059257081, + "version": 1, + "versionNonce": 1128466621, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Envoy / k8s Service", + "originalText": "Envoy / k8s Service", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el44", + "x": 518.0, + "y": 596, + "width": 124.0, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1840109256, + "version": 1, + "versionNonce": 1715412120, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el45", + "x": 518.0, + "y": 625.75, + "width": 124.0, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1554762904, + "version": 1, + "versionNonce": 941975481, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "pod 1", + "originalText": "pod 1", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el46", + "x": 648.0, + "y": 596, + "width": 124.0, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 594130309, + "version": 1, + "versionNonce": 2119634400, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el47", + "x": 648.0, + "y": 625.75, + "width": 124.0, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 390452953, + "version": 1, + "versionNonce": 202363286, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "pod 2", + "originalText": "pod 2", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el48", + "x": 510, + "y": 675, + "width": 270, + "height": 11.25, + "angle": 0, + "strokeColor": "#1971c2", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 470939446, + "version": 1, + "versionNonce": 656448480, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "KEDA . pods now = 2", + "originalText": "KEDA . pods now = 2", + "fontSize": 9, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 8, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el49", + "x": 460, + "y": 558, + "width": 185.0, + "height": 2, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 687117442, + "version": 1, + "versionNonce": 1813163241, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 185.0, + 2 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el50", + "x": 805, + "y": 528, + "width": 290, + "height": 180, + "angle": 0, + "strokeColor": "#fd7e14", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 3, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 272849425, + "version": 1, + "versionNonce": 1652563014, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el51", + "x": 815, + "y": 536, + "width": 270, + "height": 13.75, + "angle": 0, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1639042339, + "version": 1, + "versionNonce": 2010258947, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Shard 2 deployable (scaled +1)", + "originalText": "Shard 2 deployable (scaled +1)", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el52", + "x": 815, + "y": 560, + "width": 270, + "height": 28, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#eebefa", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1079815405, + "version": 1, + "versionNonce": 49310609, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el53", + "x": 815, + "y": 567.75, + "width": 270, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 491995980, + "version": 1, + "versionNonce": 1146005449, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Envoy / k8s Service", + "originalText": "Envoy / k8s Service", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el54", + "x": 823.0, + "y": 596, + "width": 80.66666666666667, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1461042544, + "version": 1, + "versionNonce": 479112938, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el55", + "x": 823.0, + "y": 625.75, + "width": 80.66666666666667, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1260573449, + "version": 1, + "versionNonce": 1867302555, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "pod 1", + "originalText": "pod 1", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el56", + "x": 909.6666666666666, + "y": 596, + "width": 80.66666666666667, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 679282379, + "version": 1, + "versionNonce": 1948728484, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el57", + "x": 909.6666666666666, + "y": 625.75, + "width": 80.66666666666667, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 13938522, + "version": 1, + "versionNonce": 1131247331, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "pod 2", + "originalText": "pod 2", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el58", + "x": 996.3333333333334, + "y": 596, + "width": 80.66666666666667, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 767303989, + "version": 1, + "versionNonce": 457031005, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el59", + "x": 996.3333333333334, + "y": 625.75, + "width": 80.66666666666667, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1281810601, + "version": 1, + "versionNonce": 854316682, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "pod 3", + "originalText": "pod 3", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el60", + "x": 815, + "y": 675, + "width": 270, + "height": 11.25, + "angle": 0, + "strokeColor": "#1971c2", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 656439678, + "version": 1, + "versionNonce": 1605947757, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "KEDA . pods now = 3", + "originalText": "KEDA . pods now = 3", + "fontSize": 9, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 8, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el61", + "x": 460, + "y": 558, + "width": 490.0, + "height": 2, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 693847830, + "version": 1, + "versionNonce": 2456274, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 490.0, + 2 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el62", + "x": 1110, + "y": 528, + "width": 290, + "height": 180, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1392239676, + "version": 1, + "versionNonce": 2098545542, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el63", + "x": 1120, + "y": 536, + "width": 270, + "height": 13.75, + "angle": 0, + "strokeColor": "#212529", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 83651971, + "version": 1, + "versionNonce": 480468220, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Shard 3 deployable", + "originalText": "Shard 3 deployable", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el64", + "x": 1120, + "y": 560, + "width": 270, + "height": 28, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#eebefa", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1558990518, + "version": 1, + "versionNonce": 1320763136, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el65", + "x": 1120, + "y": 567.75, + "width": 270, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1028439864, + "version": 1, + "versionNonce": 248786715, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Envoy / k8s Service", + "originalText": "Envoy / k8s Service", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el66", + "x": 1128.0, + "y": 596, + "width": 124.0, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1034535594, + "version": 1, + "versionNonce": 338258952, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el67", + "x": 1128.0, + "y": 625.75, + "width": 124.0, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 367878762, + "version": 1, + "versionNonce": 2087313122, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "pod 1", + "originalText": "pod 1", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el68", + "x": 1258.0, + "y": 596, + "width": 124.0, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 297265481, + "version": 1, + "versionNonce": 540137297, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el69", + "x": 1258.0, + "y": 625.75, + "width": 124.0, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 551437150, + "version": 1, + "versionNonce": 2041322271, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "pod 2", + "originalText": "pod 2", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el70", + "x": 1120, + "y": 675, + "width": 270, + "height": 11.25, + "angle": 0, + "strokeColor": "#1971c2", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 709214892, + "version": 1, + "versionNonce": 1138409540, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "KEDA . pods now = 2", + "originalText": "KEDA . pods now = 2", + "fontSize": 9, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 8, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el71", + "x": 460, + "y": 558, + "width": 795.0, + "height": 2, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1817363616, + "version": 1, + "versionNonce": 909666338, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 795.0, + 2 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el72", + "x": 80, + "y": 695, + "width": 400, + "height": 12.5, + "angle": 0, + "strokeColor": "#9c36b5", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 863937248, + "version": 1, + "versionNonce": 1338811291, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "autoscaling: storage-bound -> KEDA pinned at S.R; optional night R->1 (00-05) saves ~10%", + "originalText": "autoscaling: storage-bound -> KEDA pinned at S.R; optional night R->1 (00-05) saves ~10%", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el73", + "x": 60, + "y": 740, + "width": 1400, + "height": 230, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#f8f9fa", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1713658875, + "version": 1, + "versionNonce": 1603828850, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el74", + "x": 80, + "y": 748, + "width": 1360, + "height": 16.25, + "angle": 0, + "strokeColor": "#212529", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1881625506, + "version": 1, + "versionNonce": 1939118224, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Tenant B . Search / Ranking (1 TB / 80k peak - compute-bound - S=4, R=2, V=2)", + "originalText": "Tenant B . Search / Ranking (1 TB / 80k peak - compute-bound - S=4, R=2, V=2)", + "fontSize": 13, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 11, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el75", + "x": 80, + "y": 778, + "width": 160, + "height": 60, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffd43b", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 519709080, + "version": 1, + "versionNonce": 1064746526, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el76", + "x": 80, + "y": 800.5, + "width": 160, + "height": 15.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 965067728, + "version": 1, + "versionNonce": 274989149, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Search Svc", + "originalText": "Search Svc", + "fontSize": 12, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 10, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el77", + "x": 260, + "y": 778, + "width": 200, + "height": 60, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#99e9f2", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1452066460, + "version": 1, + "versionNonce": 90341466, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el78", + "x": 260, + "y": 794.25, + "width": 200, + "height": 27.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 988335252, + "version": 1, + "versionNonce": 945826487, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Client SDK\nkey -> shard -> pod", + "originalText": "Client SDK\nkey -> shard -> pod", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el79", + "x": 240, + "y": 808, + "width": 20, + "height": 0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 30884439, + "version": 1, + "versionNonce": 304912983, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 20, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el80", + "x": 500, + "y": 778, + "width": 290, + "height": 180, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 252860897, + "version": 1, + "versionNonce": 983297493, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el81", + "x": 510, + "y": 786, + "width": 270, + "height": 13.75, + "angle": 0, + "strokeColor": "#212529", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 289482183, + "version": 1, + "versionNonce": 134917628, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Shard 1 deployable", + "originalText": "Shard 1 deployable", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el82", + "x": 510, + "y": 810, + "width": 270, + "height": 28, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#eebefa", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1419179581, + "version": 1, + "versionNonce": 304330004, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el83", + "x": 510, + "y": 817.75, + "width": 270, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1022222126, + "version": 1, + "versionNonce": 1196050748, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Envoy / k8s Service", + "originalText": "Envoy / k8s Service", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el84", + "x": 518.0, + "y": 846, + "width": 80.66666666666667, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 2084839400, + "version": 1, + "versionNonce": 920140091, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el85", + "x": 518.0, + "y": 875.75, + "width": 80.66666666666667, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 568275056, + "version": 1, + "versionNonce": 2030106618, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "pod 1", + "originalText": "pod 1", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el86", + "x": 604.6666666666666, + "y": 846, + "width": 80.66666666666667, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1043665193, + "version": 1, + "versionNonce": 2031403297, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el87", + "x": 604.6666666666666, + "y": 875.75, + "width": 80.66666666666667, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1748309235, + "version": 1, + "versionNonce": 817804384, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "pod 2", + "originalText": "pod 2", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el88", + "x": 691.3333333333334, + "y": 846, + "width": 80.66666666666667, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 405126229, + "version": 1, + "versionNonce": 416314668, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el89", + "x": 691.3333333333334, + "y": 875.75, + "width": 80.66666666666667, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1851350740, + "version": 1, + "versionNonce": 1521696480, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "pod 3", + "originalText": "pod 3", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el90", + "x": 510, + "y": 925, + "width": 270, + "height": 11.25, + "angle": 0, + "strokeColor": "#1971c2", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1819256338, + "version": 1, + "versionNonce": 1765670845, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "KEDA . pods now = 3", + "originalText": "KEDA . pods now = 3", + "fontSize": 9, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 8, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el91", + "x": 460, + "y": 808, + "width": 185.0, + "height": 2, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 2005855668, + "version": 1, + "versionNonce": 232663463, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 185.0, + 2 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el92", + "x": 805, + "y": 778, + "width": 290, + "height": 180, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 422701551, + "version": 1, + "versionNonce": 260329456, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el93", + "x": 815, + "y": 786, + "width": 270, + "height": 13.75, + "angle": 0, + "strokeColor": "#212529", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1729245243, + "version": 1, + "versionNonce": 1457293576, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Shard 2 deployable", + "originalText": "Shard 2 deployable", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el94", + "x": 815, + "y": 810, + "width": 270, + "height": 28, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#eebefa", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 469306920, + "version": 1, + "versionNonce": 1067970821, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el95", + "x": 815, + "y": 817.75, + "width": 270, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 822873089, + "version": 1, + "versionNonce": 816941040, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "Envoy / k8s Service", + "originalText": "Envoy / k8s Service", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el96", + "x": 823.0, + "y": 846, + "width": 80.66666666666667, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1926780542, + "version": 1, + "versionNonce": 602078389, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el97", + "x": 823.0, + "y": 875.75, + "width": 80.66666666666667, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1811967842, + "version": 1, + "versionNonce": 788075127, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "pod 1", + "originalText": "pod 1", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el98", + "x": 909.6666666666666, + "y": 846, + "width": 80.66666666666667, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1196342298, + "version": 1, + "versionNonce": 1986972438, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el99", + "x": 909.6666666666666, + "y": 875.75, + "width": 80.66666666666667, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1072910528, + "version": 1, + "versionNonce": 323775635, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "text", + "text": "pod 2", + "originalText": "pod 2", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el100", + "x": 996.3333333333334, + "y": 846, + "width": 80.66666666666667, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1903232053, + "version": 1, + "versionNonce": 420514790, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067455, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el101", + "x": 996.3333333333334, + "y": 875.75, + "width": 80.66666666666667, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 217275225, + "version": 1, + "versionNonce": 63384489, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "pod 3", + "originalText": "pod 3", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el102", + "x": 815, + "y": 925, + "width": 270, + "height": 11.25, + "angle": 0, + "strokeColor": "#1971c2", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 400560568, + "version": 1, + "versionNonce": 1015242177, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "KEDA . pods now = 3", + "originalText": "KEDA . pods now = 3", + "fontSize": 9, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 8, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el103", + "x": 460, + "y": 808, + "width": 490.0, + "height": 2, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 714300771, + "version": 1, + "versionNonce": 1745535099, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 490.0, + 2 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el104", + "x": 80, + "y": 945, + "width": 400, + "height": 12.5, + "angle": 0, + "strokeColor": "#9c36b5", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 2085812760, + "version": 1, + "versionNonce": 2067418687, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "autoscaling: compute-bound -> KEDA min=R, max=peak; signal RPS+p99 (~40% saving over peak-static)", + "originalText": "autoscaling: compute-bound -> KEDA min=R, max=peak; signal RPS+p99 (~40% saving over peak-static)", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el105", + "x": 60, + "y": 990, + "width": 1400, + "height": 230, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#f8f9fa", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 918037634, + "version": 1, + "versionNonce": 1722454918, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el106", + "x": 80, + "y": 998, + "width": 1360, + "height": 16.25, + "angle": 0, + "strokeColor": "#212529", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 251837137, + "version": 1, + "versionNonce": 707111049, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "Tenant C . Ads Service (200 GB / 30k peak - compute-bound - S=2, R=2, V=1)", + "originalText": "Tenant C . Ads Service (200 GB / 30k peak - compute-bound - S=2, R=2, V=1)", + "fontSize": 13, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 11, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el107", + "x": 80, + "y": 1028, + "width": 160, + "height": 60, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffd43b", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1627677156, + "version": 1, + "versionNonce": 9257256, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el108", + "x": 80, + "y": 1050.5, + "width": 160, + "height": 15.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1676848677, + "version": 1, + "versionNonce": 1139038461, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "Ads Svc", + "originalText": "Ads Svc", + "fontSize": 12, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 10, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el109", + "x": 260, + "y": 1028, + "width": 200, + "height": 60, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#99e9f2", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1954246075, + "version": 1, + "versionNonce": 1225136115, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el110", + "x": 260, + "y": 1044.25, + "width": 200, + "height": 27.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1816803307, + "version": 1, + "versionNonce": 2090237818, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "Client SDK\nkey -> shard -> pod", + "originalText": "Client SDK\nkey -> shard -> pod", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el111", + "x": 240, + "y": 1058, + "width": 20, + "height": 0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 664847320, + "version": 1, + "versionNonce": 815605133, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 20, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el112", + "x": 500, + "y": 1028, + "width": 290, + "height": 180, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1274350419, + "version": 1, + "versionNonce": 935018221, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el113", + "x": 510, + "y": 1036, + "width": 270, + "height": 13.75, + "angle": 0, + "strokeColor": "#212529", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 251183831, + "version": 1, + "versionNonce": 261810764, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "Shard 1 deployable", + "originalText": "Shard 1 deployable", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el114", + "x": 510, + "y": 1060, + "width": 270, + "height": 28, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#eebefa", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1346922427, + "version": 1, + "versionNonce": 245522988, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el115", + "x": 510, + "y": 1067.75, + "width": 270, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 215359683, + "version": 1, + "versionNonce": 2047791083, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "Envoy / k8s Service", + "originalText": "Envoy / k8s Service", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el116", + "x": 518.0, + "y": 1096, + "width": 124.0, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 676168422, + "version": 1, + "versionNonce": 244295905, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el117", + "x": 518.0, + "y": 1125.75, + "width": 124.0, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 344076116, + "version": 1, + "versionNonce": 798112151, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "pod 1", + "originalText": "pod 1", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el118", + "x": 648.0, + "y": 1096, + "width": 124.0, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 294296874, + "version": 1, + "versionNonce": 291889681, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el119", + "x": 648.0, + "y": 1125.75, + "width": 124.0, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1010193047, + "version": 1, + "versionNonce": 1734202800, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "pod 2", + "originalText": "pod 2", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el120", + "x": 510, + "y": 1175, + "width": 270, + "height": 11.25, + "angle": 0, + "strokeColor": "#1971c2", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 514909067, + "version": 1, + "versionNonce": 1057486869, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "KEDA . pods now = 2", + "originalText": "KEDA . pods now = 2", + "fontSize": 9, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 8, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el121", + "x": 460, + "y": 1058, + "width": 185.0, + "height": 2, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 170689151, + "version": 1, + "versionNonce": 352116053, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 185.0, + 2 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el122", + "x": 805, + "y": 1028, + "width": 290, + "height": 180, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1800557284, + "version": 1, + "versionNonce": 1358798144, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el123", + "x": 815, + "y": 1036, + "width": 270, + "height": 13.75, + "angle": 0, + "strokeColor": "#212529", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1119980131, + "version": 1, + "versionNonce": 877286585, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "Shard 2 deployable", + "originalText": "Shard 2 deployable", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el124", + "x": 815, + "y": 1060, + "width": 270, + "height": 28, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#eebefa", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1349409435, + "version": 1, + "versionNonce": 1025148382, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el125", + "x": 815, + "y": 1067.75, + "width": 270, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1140805650, + "version": 1, + "versionNonce": 1699887271, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "Envoy / k8s Service", + "originalText": "Envoy / k8s Service", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el126", + "x": 823.0, + "y": 1096, + "width": 124.0, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 562117926, + "version": 1, + "versionNonce": 1288477635, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el127", + "x": 823.0, + "y": 1125.75, + "width": 124.0, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1963764598, + "version": 1, + "versionNonce": 1357970699, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "pod 1", + "originalText": "pod 1", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el128", + "x": 953.0, + "y": 1096, + "width": 124.0, + "height": 72, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff4e6", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 311570308, + "version": 1, + "versionNonce": 40009588, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el129", + "x": 953.0, + "y": 1125.75, + "width": 124.0, + "height": 12.5, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1968321320, + "version": 1, + "versionNonce": 429416214, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "pod 2", + "originalText": "pod 2", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el130", + "x": 815, + "y": 1175, + "width": 270, + "height": 11.25, + "angle": 0, + "strokeColor": "#1971c2", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 314652385, + "version": 1, + "versionNonce": 915490796, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "KEDA . pods now = 2", + "originalText": "KEDA . pods now = 2", + "fontSize": 9, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 8, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el131", + "x": 460, + "y": 1058, + "width": 490.0, + "height": 2, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1139027120, + "version": 1, + "versionNonce": 568896617, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 490.0, + 2 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el132", + "x": 80, + "y": 1195, + "width": 400, + "height": 12.5, + "angle": 0, + "strokeColor": "#9c36b5", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1498981530, + "version": 1, + "versionNonce": 295456420, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "autoscaling: compute-bound . V=1 (rollback via GCS reload, not sub-second)", + "originalText": "autoscaling: compute-bound . V=1 (rollback via GCS reload, not sub-second)", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el133", + "x": 360, + "y": 470, + "width": 0, + "height": 58, + "angle": 0, + "strokeColor": "#e8590c", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1049193573, + "version": 1, + "versionNonce": 1587106771, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 0, + 58 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el134", + "x": 160, + "y": 478, + "width": 420, + "height": 12.5, + "angle": 0, + "strokeColor": "#e8590c", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1224011539, + "version": 1, + "versionNonce": 677517497, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "watches topology + active version (same for every tenant)", + "originalText": "watches topology + active version (same for every tenant)", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el135", + "x": 720, + "y": 470, + "width": 75.0, + "height": 58, + "angle": 0, + "strokeColor": "#e8590c", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1881988385, + "version": 1, + "versionNonce": 1299301223, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -75.0, + 58 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el136", + "x": 720, + "y": 470, + "width": 75, + "height": 58, + "angle": 0, + "strokeColor": "#e8590c", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 33599985, + "version": 1, + "versionNonce": 1285821931, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -75, + 58 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el137", + "x": 680, + "y": 478, + "width": 260, + "height": 12.5, + "angle": 0, + "strokeColor": "#e8590c", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 444900635, + "version": 1, + "versionNonce": 576775952, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "assigns shard + version", + "originalText": "assigns shard + version", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el138", + "x": 510, + "y": 260, + "width": 65, + "height": 336, + "angle": 0, + "strokeColor": "#2f9e44", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1135872496, + "version": 1, + "versionNonce": 495762349, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 65, + 336 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "el139", + "x": 300, + "y": 460, + "width": 420, + "height": 12.5, + "angle": 0, + "strokeColor": "#2f9e44", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 459716010, + "version": 1, + "versionNonce": 667643691, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "data-loader pulls files from GCS (same for every tenant)", + "originalText": "data-loader pulls files from GCS (same for every tenant)", + "fontSize": 10, + "fontFamily": 1, + "textAlign": "left", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + }, + { + "id": "el140", + "x": 60, + "y": 1240, + "width": 1400, + "height": 60, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#fff9db", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1169726682, + "version": 1, + "versionNonce": 1210133980, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "rectangle" + }, + { + "id": "el141", + "x": 60, + "y": 1256.25, + "width": 1400, + "height": 27.5, + "angle": 0, + "strokeColor": "#5c3d00", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 904647470, + "version": 1, + "versionNonce": 1472659627, + "isDeleted": false, + "boundElements": [], + "updated": 1779944067456, + "link": null, + "locked": false, + "type": "text", + "text": "Legend: solid = SDK routing / control flow dashed orange = control plane writes/assigns dashed green = data-loader pulls from GCS\nEach pod contains: [ data-loader (engine-agnostic) + engine (RocksDB pluggable) + read server (TCP) ] . Tenants share Control Plane + Producer + GCS; per-tenant data plane is isolated.", + "originalText": "Legend: solid = SDK routing / control flow dashed orange = control plane writes/assigns dashed green = data-loader pulls from GCS\nEach pod contains: [ data-loader (engine-agnostic) + engine (RocksDB pluggable) + read server (TCP) ] . Tenants share Control Plane + Producer + GCS; per-tenant data plane is isolated.", + "fontSize": 11, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "top", + "baseline": 9, + "containerId": null, + "lineHeight": 1.25 + } + ], + "appState": { + "viewBackgroundColor": "#ffffff", + "gridSize": null + }, + "files": {} +} \ No newline at end of file diff --git a/py-sdk/mnemosyne_producer/README.md b/py-sdk/mnemosyne_producer/README.md new file mode 100644 index 00000000..bef2392e --- /dev/null +++ b/py-sdk/mnemosyne_producer/README.md @@ -0,0 +1,135 @@ +# `mnemosyne_producer` + +PySpark batch job — the **Producer** layer of Mnemosyne. Reads parquet, shards +by `crc32(key) % S`, encodes per-entity payloads as `persist.Data` protobuf, +and writes per-shard sorted RocksDB SST files plus a version manifest. + +``` +parquet (N files) ──► Spark SQL: project + key + shard + │ + ▼ repartition by shard, sort within partition + mapPartitions + │ + ▼ + ┌──────────────────────────────────────────┐ + │ per-Spark-partition: │ + │ RowEncoder → persist.Data → bytes │ + │ rocksdict.SstFileWriter (sorted insert)│ + └──────────────────────────────────────────┘ + │ + ▼ + ///data-.sst + //_manifest.json +``` + +## Key design choices + +| Decision | Choice | Why | +|---|---|---| +| RocksDB key | `concat_ws('\|', key_cols)` UTF-8 | matches existing `getKeyString` convention; readable; deterministic | +| RocksDB value | one entry per **entity**, value = `persist.Data` proto bytes | single point-lookup serves all features for an entity (optimal WORM read) | +| Shard hash | `crc32(IEEE) % S` | matches Spark SQL `crc32()`, Python `binascii.crc32`, Go `crc32.ChecksumIEEE` | +| Compression | none at SST level (RocksDB block compression is engine-side) | keep producer simple; serving engine picks | +| Sorted ingest | `sortWithinPartitions("__mnemo_shard", "__mnemo_key")` | required by `SstFileWriter.put` (sorted-insert contract) | +| SSTs per shard | possibly multiple (one per partition that holds rows for that shard) | RocksDB `IngestExternalFile` handles overlapping SSTs | + +## Inputs + +* **Config** (the onboarding JSON, see `examples/geo_config.json`) — declares the + entity, key columns, feature-group/feature/source-column/default-value/data-type. +* **Input parquet** — single path, directory, or glob. All files must contain + every column referenced by the config (key columns + feature source columns). +* **`--num-shards`** — total shards `S`. +* **`--version-id`** — output namespace (e.g. `20260622_r1` = `date_runOfDay`). + +## Output layout + +``` +// + 00000/data-.sst + 00001/data-.sst + ... + _manifest.json (schema, num_shards, per-shard files + sha256 + row counts) +``` + +`_manifest.json` is what the **Control Plane** consumes to validate coverage +(every shard present, row counts ≥ thresholds, checksums recorded) before +promoting the version to `activeVersion`. + +## Local / Databricks usage + +Cluster libs (Databricks: cluster init script or workspace library install): + +``` +pip install rocksdict protobuf pyspark # + bharatml_commons (existing internal pkg) +``` + +Submit: + +``` +spark-submit --py-files py-sdk/bharatml_commons.zip \ + -m mnemosyne_producer.job \ + --input gs://.../ds_dbc_ofs_catalog__geo/*.snappy.parquet \ + --config py-sdk/mnemosyne_producer/examples/geo_config.json \ + --output dbfs:/mnt/mnemo/store=catalog__user_geohash_1_3 \ + --num-shards 10 \ + --version-id 20260622_r1 +``` + +`--shuffle-partitions` is exposed as a tunable for very large inputs (it +controls the pre-repartition shuffle width; the final stage always uses +`--num-shards`). + +## Smoke test (no Spark) + +``` +python examples/smoke_test.py +``` + +Validates: + +* config parses +* `RowEncoder` produces decodable `persist.Data` bytes +* defaults applied for null source columns +* `crc32(key) % S` is in range and deterministic + +## Module layout + +``` +mnemosyne_producer/ +├── config.py parse onboarding JSON → Config(entity_label, key_columns, feature_groups[]) +├── sharding.py KEY_SEP="|" · make_key() · shard_id() · crc32_ieee() +├── encoder.py RowEncoder: row dict → persist.Data → bytes (per-FG typed Values) +├── sst_writer.py write_partition_ssts(): per-Spark-partition rocksdict.SstFileWriter +└── job.py PySpark entry point (parse args, build pipeline, write manifest) +``` + +## V1 scope / not yet + +* **Single source table per store.** Multi-source joins (entity features split + across tables) are deferred — caller must pre-join into one parquet input. +* **Vectors** are encoder-supported but unexercised by the example config. +* **No upload step** — output goes to whatever path you pass (local FS, DBFS + mount, etc.). For raw `gs://` writes, add an upload hook inside + `sst_writer._finalize`. +* **Pluggable file format** for the Databricks job is V2 in the LLD; V1 emits + `persist.Data` proto bytes only. + +## Where this fits in Mnemosyne + +``` +THIS JOB ────► GCS (versioned SSTs + manifest) + │ +Control Plane watches GCS ─────┘ + mints version, writes shard→pod assignment in etcd + │ + ▼ +data-loader (engine-agnostic sidecar on each pod) + pulls assigned SSTs, calls engine.bulk_ingest() + re-PUTs PodData with WarmVersions += [vId] + │ + ▼ +Read server (Rust + TCP) ─► serves point lookups by key +``` + +See the [Mnemosyne LLD](https://meesho.atlassian.net/wiki/spaces/EW/pages/5379850241) for the full platform design. diff --git a/py-sdk/mnemosyne_producer/examples/geo_config.json b/py-sdk/mnemosyne_producer/examples/geo_config.json new file mode 100644 index 00000000..44fb35b9 --- /dev/null +++ b/py-sdk/mnemosyne_producer/examples/geo_config.json @@ -0,0 +1,62 @@ +{ + "data": [ + { + "storage-provider": "TABLE", + "base-path": [ + { + "source-base-path": "ds_dbc_ofs.catalog__user_geohash_1_3__derived", + "data-paths": [ + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "orders_by_clicks_7_days_ewma", "source-data-column": "user_geohash_res_1__orders_by_clicks_7_days_ewma", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__clicks_by_views_14_days", "source-data-column": "user_geohash_res_3__clicks_by_views_14_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__orders_by_clicks_1_days", "source-data-column": "user_geohash_res_1__orders_by_clicks_1_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__orders_by_views_3_days", "source-data-column": "user_geohash_res_3__orders_by_views_3_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "rollup_int32", "feature-label": "user_geohash_res_3__clicks_3_days", "source-data-column": "user_geohash_res_3__clicks_3_days", "default-value": "0", "data-type": "DataTypeInt32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__orders_by_clicks_1_days", "source-data-column": "user_geohash_res_3__orders_by_clicks_1_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__per_wfr_return", "source-data-column": "user_geohash_res_1__per_wfr_return", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__nqp_by_nqd_ema", "source-data-column": "user_geohash_res_1__nqp_by_nqd_ema", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__clicks_by_views_5_days", "source-data-column": "user_geohash_res_3__clicks_by_views_5_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__clicks_by_views_7_days", "source-data-column": "user_geohash_res_3__clicks_by_views_7_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "rollup_int32", "feature-label": "user_geohash_res_1__clicks_3_days", "source-data-column": "user_geohash_res_1__clicks_3_days", "default-value": "0", "data-type": "DataTypeInt32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__orders_by_views_1_days", "source-data-column": "user_geohash_res_3__orders_by_views_1_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "orders_by_views_7_days_ewma", "source-data-column": "user_geohash_res_1__orders_by_views_7_days_ewma", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__clicks_by_views_7_days", "source-data-column": "user_geohash_res_1__clicks_by_views_7_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__orders_by_views_14_days", "source-data-column": "user_geohash_res_3__orders_by_views_14_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__orders_by_clicks_28_days","source-data-column": "user_geohash_res_3__orders_by_clicks_28_days","default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__orders_by_clicks_28_days","source-data-column": "user_geohash_res_1__orders_by_clicks_28_days","default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__nqp_by_nqd_smoothened", "source-data-column": "user_geohash_res_1__nqp_by_nqd_smoothened", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__orders_by_clicks_56_days","source-data-column": "user_geohash_res_1__orders_by_clicks_56_days","default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__orders_by_clicks_14_days","source-data-column": "user_geohash_res_1__orders_by_clicks_14_days","default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__orders_by_clicks_14_days","source-data-column": "user_geohash_res_3__orders_by_clicks_14_days","default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__orders_by_clicks_56_days","source-data-column": "user_geohash_res_3__orders_by_clicks_56_days","default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__orders_by_clicks_3_days", "source-data-column": "user_geohash_res_1__orders_by_clicks_3_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__nqp_smoothened", "source-data-column": "user_geohash_res_1__nqp_smoothened", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__orders_by_clicks_5_days", "source-data-column": "user_geohash_res_3__orders_by_clicks_5_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__orders_by_views_7_days", "source-data-column": "user_geohash_res_3__orders_by_views_7_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__orders_by_views_5_days", "source-data-column": "user_geohash_res_3__orders_by_views_5_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__clicks_by_views_1_days", "source-data-column": "user_geohash_res_3__clicks_by_views_1_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__clicks_by_views_14_days_ewma", "source-data-column": "user_geohash_res_1__clicks_by_views_14_days_ewma", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__clicks_by_views_56_days", "source-data-column": "user_geohash_res_3__clicks_by_views_56_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__clicks_by_views_1_days", "source-data-column": "user_geohash_res_1__clicks_by_views_1_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__clicks_by_views_28_days", "source-data-column": "user_geohash_res_3__clicks_by_views_28_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__orders_by_clicks_14_days_ewma", "source-data-column": "user_geohash_res_1__orders_by_clicks_14_days_ewma", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__per_return", "source-data-column": "user_geohash_res_1__per_return", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__orders_by_clicks_7_days", "source-data-column": "user_geohash_res_3__orders_by_clicks_7_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__orders_by_clicks_3_days", "source-data-column": "user_geohash_res_3__orders_by_clicks_3_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__clicks_by_views_3_days", "source-data-column": "user_geohash_res_3__clicks_by_views_3_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__orders_by_views_14_days_ewma", "source-data-column": "user_geohash_res_1__orders_by_views_14_days_ewma", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__clicks_by_views_3_days", "source-data-column": "user_geohash_res_1__clicks_by_views_3_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__clicks_by_views_56_days", "source-data-column": "user_geohash_res_1__clicks_by_views_56_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__per_qr_return", "source-data-column": "user_geohash_res_1__per_qr_return", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__nqp_ema", "source-data-column": "user_geohash_res_1__nqp_ema", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_1__clicks_by_views_28_days", "source-data-column": "user_geohash_res_1__clicks_by_views_28_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "user_geohash_res_3__orders_by_views_28_days", "source-data-column": "user_geohash_res_3__orders_by_views_28_days", "default-value": "0", "data-type": "DataTypeFP32" }, + { "entity-label": "catalog__user_geohash_1_3", "feature-group-label": "derived_fp32", "feature-label": "clicks_by_views_7_days_ewma", "source-data-column": "user_geohash_res_1__clicks_by_views_7_days_ewma", "default-value": "0", "data-type": "DataTypeFP32" } + ] + } + ] + } + ], + "keys": { + "catalog__user_geohash_1_3": ["geohash_1_3_id", "catalog_id"] + } +} diff --git a/py-sdk/mnemosyne_producer/examples/smoke_test.py b/py-sdk/mnemosyne_producer/examples/smoke_test.py new file mode 100644 index 00000000..bb73875d --- /dev/null +++ b/py-sdk/mnemosyne_producer/examples/smoke_test.py @@ -0,0 +1,96 @@ +""" +No-Spark smoke test for the encoder + sharding logic. + +Run: + cd py-sdk/mnemosyne_producer + pip install -e . --no-deps # or just add the parent dir to PYTHONPATH + PYTHONPATH=../bharatml_commons python examples/smoke_test.py + +Asserts: + * Config parses the sample geo_config.json. + * RowEncoder turns a synthetic row into persist.Data bytes round-trippable + via the existing persist_pb2 module. + * shard_id is deterministic and in [0, num_shards). + * Default values are applied when source columns are null. +""" +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + + +def main() -> int: + here = Path(__file__).resolve().parent + pkg_root = here.parent + sys.path.insert(0, str(pkg_root)) + + from mnemosyne_producer.config import Config + from mnemosyne_producer.encoder import RowEncoder + from mnemosyne_producer.sharding import make_key, shard_id + + cfg = Config.from_json(str(here / "geo_config.json")) + assert cfg.entity_label == "catalog__user_geohash_1_3" + assert cfg.key_columns == ["geohash_1_3_id", "catalog_id"] + fg_labels = [fg.label for fg in cfg.feature_groups] + assert "derived_fp32" in fg_labels + assert "rollup_int32" in fg_labels + print(f"config OK: entity={cfg.entity_label} keys={cfg.key_columns} " + f"fgs={fg_labels} features_total={sum(len(fg.features) for fg in cfg.feature_groups)}") + + # Synthetic row with a couple of features filled and most as nulls (defaults apply). + row = { + "geohash_1_3_id": "abc123", + "catalog_id": 99, + # one filled FP32 source col: + "user_geohash_res_1__orders_by_clicks_7_days_ewma": 0.42, + # one filled Int32 source col: + "user_geohash_res_3__clicks_3_days": 7, + # the rest are None -> defaults from config (= 0) + } + # Sharding + key + key_str = make_key(cfg.key_columns, row) + assert key_str == "abc123|99", key_str + sid = shard_id(key_str, 10) + assert 0 <= sid < 10 + print(f"key={key_str!r} shard_id={sid}/10") + + # Encode + encoder = RowEncoder(cfg) + blob = encoder.encode(row) + assert isinstance(blob, bytes) and len(blob) > 0 + + # Decode & verify round-trip + from bharatml_commons.proto.persist.persist_pb2 import Data + data = Data() + data.ParseFromString(blob) + assert list(data.key_values) == ["abc123", "99"] + assert len(data.feature_values) == len(cfg.feature_groups) + + # Find derived_fp32 block by position + fg_idx = {fg.label: i for i, fg in enumerate(cfg.feature_groups)} + fp32_block = data.feature_values[fg_idx["derived_fp32"]].values + int32_block = data.feature_values[fg_idx["rollup_int32"]].values + print(f"derived_fp32: {len(fp32_block.fp32_values)} fp32 values " + f"(first 3 = {list(fp32_block.fp32_values[:3])})") + print(f"rollup_int32: {len(int32_block.int32_values)} int32 values " + f"(values = {list(int32_block.int32_values)})") + + # The filled features should show non-default values: + derived = next(fg for fg in cfg.feature_groups if fg.label == "derived_fp32") + filled_idx = [i for i, f in enumerate(derived.features) + if f.source_column == "user_geohash_res_1__orders_by_clicks_7_days_ewma"][0] + assert abs(fp32_block.fp32_values[filled_idx] - 0.42) < 1e-6 + + rollup = next(fg for fg in cfg.feature_groups if fg.label == "rollup_int32") + filled_idx2 = [i for i, f in enumerate(rollup.features) + if f.source_column == "user_geohash_res_3__clicks_3_days"][0] + assert int32_block.int32_values[filled_idx2] == 7 + + print("ALL GOOD") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/py-sdk/mnemosyne_producer/mnemosyne_producer/__init__.py b/py-sdk/mnemosyne_producer/mnemosyne_producer/__init__.py new file mode 100644 index 00000000..3dc1f76b --- /dev/null +++ b/py-sdk/mnemosyne_producer/mnemosyne_producer/__init__.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/py-sdk/mnemosyne_producer/mnemosyne_producer/config.py b/py-sdk/mnemosyne_producer/mnemosyne_producer/config.py new file mode 100644 index 00000000..ed947e52 --- /dev/null +++ b/py-sdk/mnemosyne_producer/mnemosyne_producer/config.py @@ -0,0 +1,158 @@ +""" +Parse the onboarding JSON config into a typed structure. + +The input JSON looks like: + + { + "data": [ + { + "storage-provider": "TABLE", + "base-path": [ + { + "source-base-path": "ds_dbc_ofs.catalog__user_geohash_1_3__derived", + "data-paths": [ + { "entity-label": "catalog__user_geohash_1_3", + "feature-group-label": "derived_fp32", + "feature-label": "orders_by_clicks_7_days_ewma", + "source-data-column": "user_geohash_res_1__orders_by_clicks_7_days_ewma", + "default-value": "0", + "data-type": "DataTypeFP32" }, + ... + ] + } + ] + } + ], + "keys": { "catalog__user_geohash_1_3": ["geohash_1_3_id", "catalog_id"] } + } + +Constraints / V1 scope: + * Single entity_label per config (multi-entity later — one config per store). + * All features within one feature-group-label share the same data-type. + * Feature-group + feature order is preserved from the JSON (insertion order). + Producer and consumer must agree on this order; we emit the schema to the + output manifest so the consumer can decode positionally. +""" +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Any, Dict, List + + +@dataclass(frozen=True) +class FeatureSpec: + label: str # e.g., "orders_by_clicks_7_days_ewma" + source_column: str # e.g., "user_geohash_res_1__orders_by_clicks_7_days_ewma" + default_value: str # raw default from JSON; cast at encode time + data_type: str # e.g., "DataTypeFP32" + + +@dataclass +class FeatureGroupSpec: + label: str + features: List[FeatureSpec] = field(default_factory=list) + + @property + def data_type(self) -> str: + if not self.features: + return "" + return self.features[0].data_type + + def assert_uniform_dtype(self) -> None: + dt = self.data_type + for f in self.features: + if f.data_type != dt: + raise ValueError( + f"feature-group '{self.label}' has mixed data-types: " + f"expected {dt}, got {f.data_type} on feature {f.label!r}" + ) + + +@dataclass +class Config: + entity_label: str + key_columns: List[str] + feature_groups: List[FeatureGroupSpec] + # The original source-base-path declared in the JSON (informational; the actual + # input is supplied at job submit time via --input). + source_base_paths: List[str] = field(default_factory=list) + + def all_source_columns(self) -> List[str]: + cols: List[str] = [] + seen = set() + for c in self.key_columns: + if c not in seen: + cols.append(c); seen.add(c) + for fg in self.feature_groups: + for f in fg.features: + if f.source_column not in seen: + cols.append(f.source_column); seen.add(f.source_column) + return cols + + def schema_dict(self) -> Dict[str, Any]: + """Schema written into the global manifest so consumers can decode positionally.""" + return { + "entity_label": self.entity_label, + "key_columns": list(self.key_columns), + "feature_groups": [ + { + "label": fg.label, + "data_type": fg.data_type, + "features": [ + { + "label": f.label, + "source_column": f.source_column, + "default_value": f.default_value, + "data_type": f.data_type, + } + for f in fg.features + ], + } + for fg in self.feature_groups + ], + "source_base_paths": list(self.source_base_paths), + } + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> "Config": + keys = d.get("keys") or {} + if len(keys) != 1: + raise ValueError("Config 'keys' must contain exactly one entity_label (V1).") + entity_label = next(iter(keys)) + key_columns = list(keys[entity_label]) + + fg_map: Dict[str, FeatureGroupSpec] = {} + source_paths: List[str] = [] + + for block in d.get("data", []): + for base in block.get("base-path", []) or []: + source_paths.append(base.get("source-base-path", "")) + for dp in base.get("data-paths", []) or []: + if dp.get("entity-label") != entity_label: + # Skip rows that belong to a different entity (defensive). + continue + fg_label = dp["feature-group-label"] + fg = fg_map.setdefault(fg_label, FeatureGroupSpec(label=fg_label)) + fg.features.append(FeatureSpec( + label=dp["feature-label"], + source_column=dp["source-data-column"], + default_value=str(dp.get("default-value", "0")), + data_type=dp["data-type"], + )) + + feature_groups = list(fg_map.values()) + for fg in feature_groups: + fg.assert_uniform_dtype() + + return cls( + entity_label=entity_label, + key_columns=key_columns, + feature_groups=feature_groups, + source_base_paths=source_paths, + ) + + @classmethod + def from_json(cls, path: str) -> "Config": + with open(path) as f: + return cls.from_dict(json.load(f)) diff --git a/py-sdk/mnemosyne_producer/mnemosyne_producer/encoder.py b/py-sdk/mnemosyne_producer/mnemosyne_producer/encoder.py new file mode 100644 index 00000000..3a1b775f --- /dev/null +++ b/py-sdk/mnemosyne_producer/mnemosyne_producer/encoder.py @@ -0,0 +1,175 @@ +""" +Row → persist.Data → bytes. + +For each entity row we build a persist.Data proto containing: + * key_values: the entity's key columns as strings (e.g. ["abc123", "99"]). + * feature_values: one `FeatureValues` per feature-group, in the order + declared in the config. Each FG's `Values` holds a single + typed parallel array (fp32_values / int32_values / ...) in + feature-label order. + +Null source values are replaced by the per-feature default declared in the +config (cast to the target type once at encoder-construction time). + +We pre-compute as much as possible at __init__ (data-type branches, casters, +default arrays) so the hot path is tight. + +Vectors are supported for the existing proto vector types; for V1 we only +exercise scalar paths from the user's geo config. +""" +from __future__ import annotations + +from typing import Any, Callable, Dict, List + +from .config import Config, FeatureGroupSpec, FeatureSpec + + +# Group data types by the proto field they populate. +_FP32_TYPES = {"DataTypeFP8E5M2", "DataTypeFP8E4M3", "DataTypeFP16", "DataTypeFP32"} +_FP64_TYPES = {"DataTypeFP64"} +_I32_TYPES = {"DataTypeInt8", "DataTypeInt16", "DataTypeInt32"} +_I64_TYPES = {"DataTypeInt64"} +_U32_TYPES = {"DataTypeUint8", "DataTypeUint16", "DataTypeUint32"} +_U64_TYPES = {"DataTypeUint64"} + +_FP32_VEC = {"DataTypeFP8E5M2Vector", "DataTypeFP8E4M3Vector", + "DataTypeFP16Vector", "DataTypeFP32Vector"} +_FP64_VEC = {"DataTypeFP64Vector"} +_I32_VEC = {"DataTypeInt8Vector", "DataTypeInt16Vector", "DataTypeInt32Vector"} +_I64_VEC = {"DataTypeInt64Vector"} +_U32_VEC = {"DataTypeUint8Vector", "DataTypeUint16Vector", "DataTypeUint32Vector"} +_U64_VEC = {"DataTypeUint64Vector"} + + +def _cast_default(default_str: str, data_type: str) -> Any: + """Cast the textual default-value from JSON to the target Python type.""" + if data_type in _FP32_TYPES or data_type in _FP32_VEC: + return float(default_str or "0") + if data_type in _FP64_TYPES or data_type in _FP64_VEC: + return float(default_str or "0") + if data_type in _I32_TYPES or data_type in _I32_VEC \ + or data_type in _I64_TYPES or data_type in _I64_VEC: + return int(default_str or "0") + if data_type in _U32_TYPES or data_type in _U32_VEC \ + or data_type in _U64_TYPES or data_type in _U64_VEC: + return int(default_str or "0") + if data_type == "DataTypeString" or data_type == "DataTypeStringVector": + return default_str or "" + if data_type == "DataTypeBool" or data_type == "DataTypeBoolVector": + s = (default_str or "0").strip().lower() + return s in ("1", "true", "yes", "y", "t") + raise ValueError(f"unsupported data_type: {data_type}") + + +class _FgPlan: + """Pre-built per-feature-group encoding plan.""" + __slots__ = ("label", "data_type", "source_columns", "defaults", "is_vector") + + def __init__(self, fg: FeatureGroupSpec): + self.label = fg.label + self.data_type = fg.data_type + self.is_vector = "Vector" in fg.data_type + self.source_columns: List[str] = [f.source_column for f in fg.features] + self.defaults: List[Any] = [_cast_default(f.default_value, f.data_type) for f in fg.features] + + +class RowEncoder: + """Builds persist.Data bytes from a row dict.""" + + def __init__(self, cfg: Config): + self.cfg = cfg + self.fg_plans: List[_FgPlan] = [_FgPlan(fg) for fg in cfg.feature_groups] + + # ---------- public API ---------- + + def encode(self, row: Dict[str, Any]) -> bytes: + # Local import so the proto package is resolved on Spark executors + # (the import only happens once per partition since RowEncoder is + # constructed there). + from bharatml_commons.proto.persist.persist_pb2 import Data, FeatureValues, Values # noqa: WPS433 + + key_values = [ + ("" if row.get(c) is None else str(row.get(c))) + for c in self.cfg.key_columns + ] + + feature_values = [] + for plan in self.fg_plans: + vals = Values() + self._fill(vals, plan, row) + feature_values.append(FeatureValues(values=vals)) + + return Data(key_values=key_values, feature_values=feature_values).SerializeToString() + + # ---------- internals ---------- + + def _fill(self, values: "Any", plan: _FgPlan, row: Dict[str, Any]) -> None: + dt = plan.data_type + # Resolve per-feature value or default + # NOTE: row.get(col) returns None for missing/null; defaults already typed. + if dt in _FP32_TYPES: + arr = [self._or_default(row.get(c), d, float) for c, d in zip(plan.source_columns, plan.defaults)] + values.fp32_values.extend(arr) # proto field is double[] but holds fp32 logically + elif dt in _FP64_TYPES: + arr = [self._or_default(row.get(c), d, float) for c, d in zip(plan.source_columns, plan.defaults)] + values.fp64_values.extend(arr) + elif dt in _I32_TYPES: + arr = [self._or_default(row.get(c), d, int) for c, d in zip(plan.source_columns, plan.defaults)] + values.int32_values.extend(arr) + elif dt in _I64_TYPES: + arr = [self._or_default(row.get(c), d, int) for c, d in zip(plan.source_columns, plan.defaults)] + values.int64_values.extend(arr) + elif dt in _U32_TYPES: + arr = [self._or_default(row.get(c), d, int) for c, d in zip(plan.source_columns, plan.defaults)] + values.uint32_values.extend(arr) + elif dt in _U64_TYPES: + arr = [self._or_default(row.get(c), d, int) for c, d in zip(plan.source_columns, plan.defaults)] + values.uint64_values.extend(arr) + elif dt == "DataTypeString": + arr = [self._or_default(row.get(c), d, str) for c, d in zip(plan.source_columns, plan.defaults)] + values.string_values.extend(arr) + elif dt == "DataTypeBool": + arr = [self._or_default(row.get(c), d, bool) for c, d in zip(plan.source_columns, plan.defaults)] + values.bool_values.extend(arr) + elif plan.is_vector: + self._fill_vector(values, plan, row) + else: + raise ValueError(f"unsupported data_type: {dt}") + + def _fill_vector(self, values: "Any", plan: _FgPlan, row: Dict[str, Any]) -> None: + from bharatml_commons.proto.persist.persist_pb2 import Values, Vector # noqa: WPS433 + dt = plan.data_type + for src, default in zip(plan.source_columns, plan.defaults): + raw = row.get(src) + seq = raw if raw is not None else default # default may be scalar; vector default rarely useful + inner = Values() + if dt in _FP32_VEC: + inner.fp32_values.extend(float(x) for x in seq) + elif dt in _FP64_VEC: + inner.fp64_values.extend(float(x) for x in seq) + elif dt in _I32_VEC: + inner.int32_values.extend(int(x) for x in seq) + elif dt in _I64_VEC: + inner.int64_values.extend(int(x) for x in seq) + elif dt in _U32_VEC: + inner.uint32_values.extend(int(x) for x in seq) + elif dt in _U64_VEC: + inner.uint64_values.extend(int(x) for x in seq) + elif dt == "DataTypeStringVector": + inner.string_values.extend(str(x) for x in seq) + elif dt == "DataTypeBoolVector": + inner.bool_values.extend(bool(x) for x in seq) + else: + raise ValueError(f"unsupported vector data_type: {dt}") + values.vector.append(Vector(values=inner)) + + @staticmethod + def _or_default(val: Any, default: Any, caster: Callable[[Any], Any]) -> Any: + if val is None: + return default + try: + # Fast path: already correct type + return caster(val) + except (TypeError, ValueError): + # NaN / inf for floats → default for safety + return default diff --git a/py-sdk/mnemosyne_producer/mnemosyne_producer/job.py b/py-sdk/mnemosyne_producer/mnemosyne_producer/job.py new file mode 100644 index 00000000..32c83b65 --- /dev/null +++ b/py-sdk/mnemosyne_producer/mnemosyne_producer/job.py @@ -0,0 +1,210 @@ +""" +Mnemosyne Producer — PySpark batch job. + +Pipeline +======== + + 1. Read input parquet (one or many files). + 2. Project to only the columns needed (key cols + feature source cols). + 3. Build the composite key `__mnemo_key = concat_ws('|', key_cols...)`. + 4. Compute the shard `__mnemo_shard = crc32(__mnemo_key) % S` + using Spark SQL's `crc32()` (IEEE polynomial — identical to + `binascii.crc32` and Go's `hash/crc32.ChecksumIEEE`). + 5. Repartition by `__mnemo_shard` and `sortWithinPartitions` by + (`__mnemo_shard`, `__mnemo_key`). + 6. `mapPartitions` — encode each row to a persist.Data proto and write + SST files via `rocksdict.SstFileWriter` (sorted insert). + 7. Collect per-shard SST manifests and emit a global `_manifest.json` + for the version. + +Output layout +============= + + ///data-.sst + //_manifest.json + +Submit (local) +============== + + spark-submit \ + --packages "io.delta:delta-core_2.12:2.4.0" \ + -m mnemosyne_producer.job \ + --input /path/to/*.snappy.parquet \ + --config /path/to/geo_config.json \ + --output /tmp/mnemo_out \ + --num-shards 10 \ + --version-id 20260622_r1 +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from typing import Any, Dict, List + +from .config import Config +from .sharding import KEY_SEP + + +_SHARD_COL = "__mnemo_shard" +_KEY_COL = "__mnemo_key" + + +def parse_args(argv: List[str] | None = None) -> argparse.Namespace: + ap = argparse.ArgumentParser(description="Mnemosyne Producer (PySpark)") + ap.add_argument("--input", required=True, + help="Parquet input — file, directory, or glob (Spark-readable)") + ap.add_argument("--config", required=True, + help="Path to onboarding JSON config (entity/keys/features)") + ap.add_argument("--output", required=True, + help="Base output dir (local path or DBFS-mounted path)") + ap.add_argument("--num-shards", type=int, required=True) + ap.add_argument("--version-id", required=True, + help="Version identifier, e.g. '20260622_r1'") + ap.add_argument("--shuffle-partitions", type=int, default=None, + help="Override spark.sql.shuffle.partitions (default: leave as is)") + ap.add_argument("--app-name", default="mnemosyne-producer") + return ap.parse_args(argv) + + +def build_spark(app_name: str, shuffle_partitions: int | None): + from pyspark.sql import SparkSession + builder = SparkSession.builder.appName(app_name) + if shuffle_partitions is not None: + builder = builder.config("spark.sql.shuffle.partitions", str(shuffle_partitions)) + return builder.getOrCreate() + + +def run(args: argparse.Namespace) -> None: + from pyspark.sql import functions as F + from pyspark.sql.types import LongType + + cfg = Config.from_json(args.config) + spark = build_spark(args.app_name, args.shuffle_partitions) + + # ----- 1. Read & project ------------------------------------------------- + df = spark.read.parquet(args.input) + project_cols = cfg.all_source_columns() + missing = [c for c in project_cols if c not in df.columns] + if missing: + raise ValueError( + f"Input parquet is missing required columns: {missing[:10]}" + f"{' ...' if len(missing) > 10 else ''}" + ) + df = df.select(*project_cols) + + # ----- 2. Composite key + shard_id (vectorised in Spark SQL) ------------- + key_parts = [ + F.coalesce(F.col(c).cast("string"), F.lit("")) for c in cfg.key_columns + ] + df = df.withColumn(_KEY_COL, F.concat_ws(KEY_SEP, *key_parts)) + df = df.withColumn( + _SHARD_COL, + (F.crc32(F.col(_KEY_COL)) % F.lit(args.num_shards)).cast("int"), + ) + + # ----- 3. Repartition by shard, sort within partition -------------------- + df = ( + df.repartition(args.num_shards, F.col(_SHARD_COL)) + .sortWithinPartitions(_SHARD_COL, _KEY_COL) + ) + + # ----- 4. Encode + write SSTs in mapPartitions --------------------------- + out_base = args.output.rstrip("/") + version_id = args.version_id + cfg_dict_for_workers = cfg.schema_dict() # cheap, picklable + + def _process(rows_iter): + # Imports happen on the executor, once per partition. + from mnemosyne_producer.encoder import RowEncoder + from mnemosyne_producer.config import Config as _Config + from mnemosyne_producer.sst_writer import write_partition_ssts + cfg_local = _Config( + entity_label=cfg_dict_for_workers["entity_label"], + key_columns=cfg_dict_for_workers["key_columns"], + feature_groups=[ + _make_fg(fg_dict) for fg_dict in cfg_dict_for_workers["feature_groups"] + ], + source_base_paths=cfg_dict_for_workers.get("source_base_paths", []), + ) + encoder = RowEncoder(cfg_local) + yield from write_partition_ssts(rows_iter, encoder, out_base, version_id) + + manifests: List[Dict[str, Any]] = df.rdd.mapPartitions(_process).collect() + + # ----- 5. Global manifest ------------------------------------------------ + by_shard: Dict[int, List[Dict[str, Any]]] = {} + for m in manifests: + by_shard.setdefault(int(m["shard_id"]), []).append(m) + + global_manifest = { + "version_id": version_id, + "num_shards": args.num_shards, + "key_separator": KEY_SEP, + "value_format": { + "encoding": "protobuf", + "schema": "persist.Data (BharatMLStack/online-feature-store/pkg/proto/persist.proto)", + "fg_order": [fg["label"] for fg in cfg_dict_for_workers["feature_groups"]], + }, + "schema": cfg_dict_for_workers, + "shards": { + str(sid): { + "row_count": sum(m["rows"] for m in entries), + "files": [ + { + "path": os.path.relpath(m["path"], out_base), + "rows": m["rows"], + "bytes_logical": m["bytes_logical"], + "size_on_disk": m["size_on_disk"], + "sha256": m["sha256"], + } + for m in entries + ], + } + for sid, entries in sorted(by_shard.items()) + }, + "stats": { + "total_rows": sum(m["rows"] for m in manifests), + "total_files": len(manifests), + "shards_with_data": len(by_shard), + }, + } + + manifest_dir = os.path.join(out_base, version_id) + os.makedirs(manifest_dir, exist_ok=True) + manifest_path = os.path.join(manifest_dir, "_manifest.json") + with open(manifest_path, "w") as f: + json.dump(global_manifest, f, indent=2, default=str) + + print(f"[mnemosyne-producer] wrote {len(manifests)} SST files across " + f"{len(by_shard)} / {args.num_shards} shards " + f"({global_manifest['stats']['total_rows']:,} rows total)") + print(f"[mnemosyne-producer] global manifest: {manifest_path}") + + +def _make_fg(d: Dict[str, Any]): + """Rebuild a FeatureGroupSpec from its schema dict (executor-side).""" + from .config import FeatureGroupSpec, FeatureSpec + return FeatureGroupSpec( + label=d["label"], + features=[ + FeatureSpec( + label=f["label"], + source_column=f["source_column"], + default_value=f["default_value"], + data_type=f["data_type"], + ) + for f in d["features"] + ], + ) + + +def main(argv: List[str] | None = None) -> int: + args = parse_args(argv) + run(args) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/py-sdk/mnemosyne_producer/mnemosyne_producer/sharding.py b/py-sdk/mnemosyne_producer/mnemosyne_producer/sharding.py new file mode 100644 index 00000000..851d3159 --- /dev/null +++ b/py-sdk/mnemosyne_producer/mnemosyne_producer/sharding.py @@ -0,0 +1,41 @@ +""" +Sharding & key construction. + + shard_id = crc32_ieee(key_bytes) % num_shards + +Python's `binascii.crc32` and Spark SQL's `crc32(col)` and Go's +`hash/crc32.ChecksumIEEE` all use the IEEE 802.3 polynomial, so the shard_id is +identical across Databricks (Spark SQL), the Python encoder, and the Go/Rust +serving stack. + +Key convention: composite keys are pipe-joined and UTF-8 encoded — matches the +existing online-feature-store handler `getKeyString` in +`internal/handler/feature/retrieve.go`. +""" +from __future__ import annotations + +import binascii +from typing import Iterable, Sequence + + +KEY_SEP = "|" + + +def crc32_ieee(b: bytes) -> int: + """CRC32 (IEEE polynomial), unsigned 32-bit.""" + return binascii.crc32(b) & 0xFFFFFFFF + + +def make_key(key_columns: Sequence[str], row: dict, sep: str = KEY_SEP) -> str: + """Build the composite entity key for an entity row. Null/missing → empty string.""" + parts = [] + for c in key_columns: + v = row.get(c) + parts.append("" if v is None else str(v)) + return sep.join(parts) + + +def shard_id(key_str: str, num_shards: int) -> int: + if num_shards <= 0: + raise ValueError(f"num_shards must be positive, got {num_shards}") + return crc32_ieee(key_str.encode("utf-8")) % num_shards diff --git a/py-sdk/mnemosyne_producer/mnemosyne_producer/sst_writer.py b/py-sdk/mnemosyne_producer/mnemosyne_producer/sst_writer.py new file mode 100644 index 00000000..92c52f47 --- /dev/null +++ b/py-sdk/mnemosyne_producer/mnemosyne_producer/sst_writer.py @@ -0,0 +1,114 @@ +""" +Per-Spark-partition SST writer. + +Each Spark partition contains rows already sorted by (shard_id, key). We +iterate the partition and emit ONE SST file per (partition, shard_id) tuple; +multiple Spark partitions may contribute SSTs to the same shard, all of which +RocksDB's `IngestExternalFile` will absorb on the serving side. + +Outputs per partition: + ///data-.sst + +A small per-SST manifest dict is yielded back to the driver so we can build the +global `_manifest.json` after `collect()`. + +Notes +----- +* Output path is treated as a local-filesystem path; for cloud writes (`gs://`) + the caller is expected to be running on a worker that has a mount for that + path (Databricks DBFS) or to add an upload step. The code is structured so + that an upload hook can be added easily inside `_finalize`. +* SST file keys MUST be inserted in strictly increasing order — guaranteed by + the upstream `sortWithinPartitions("__mnemo_shard", "__mnemo_key")`. +""" +from __future__ import annotations + +import hashlib +import os +import uuid +from typing import Any, Dict, Iterator + +from .encoder import RowEncoder + + +_SHARD_COL = "__mnemo_shard" +_KEY_COL = "__mnemo_key" + + +def write_partition_ssts( + rows_iter: Iterator[Any], + encoder: RowEncoder, + out_base: str, + version_id: str, +) -> Iterator[Dict[str, Any]]: + """Consume a Spark partition iterator; emit per-shard SST manifests. + + Rows are expected as pyspark `Row` objects with `__mnemo_shard` (int) and + `__mnemo_key` (str) attributes plus the projected feature columns. + """ + try: + from rocksdict import SstFileWriter # type: ignore + except ImportError as exc: # pragma: no cover + raise RuntimeError( + "rocksdict is required on Spark workers. Install via cluster " + "library: `pip install rocksdict`." + ) from exc + + partition_uuid = uuid.uuid4().hex[:12] + + current_shard: int | None = None + writer: "SstFileWriter | None" = None + sst_path: str = "" + sha: "hashlib._Hash | None" = None + n_rows: int = 0 + n_bytes: int = 0 + + def _finalize() -> Dict[str, Any]: + """Close the active writer and emit its manifest entry.""" + nonlocal writer, current_shard, sha, n_rows, n_bytes, sst_path + assert writer is not None + assert sha is not None + writer.finish() + size = os.path.getsize(sst_path) if os.path.exists(sst_path) else None + manifest = { + "shard_id": current_shard, + "path": sst_path, + "rows": n_rows, + "bytes_logical": n_bytes, + "size_on_disk": size, + "sha256": sha.hexdigest(), + "partition_uuid": partition_uuid, + } + writer = None + return manifest + + for row in rows_iter: + # `Row` supports both attribute and dict access. + row_dict = row.asDict(recursive=False) if hasattr(row, "asDict") else dict(row) + sid = int(row_dict[_SHARD_COL]) + key_str = row_dict[_KEY_COL] + key_bytes = key_str.encode("utf-8") + value_bytes = encoder.encode(row_dict) + + if sid != current_shard: + if writer is not None: + yield _finalize() + current_shard = sid + shard_dir = os.path.join(out_base, version_id, f"{sid:05d}") + os.makedirs(shard_dir, exist_ok=True) + sst_path = os.path.join(shard_dir, f"data-{partition_uuid}.sst") + writer = SstFileWriter() + writer.open(sst_path) + sha = hashlib.sha256() + n_rows = 0 + n_bytes = 0 + + # rocksdict's SstFileWriter supports __setitem__ and put(). + writer[key_bytes] = value_bytes # type: ignore[index] + sha.update(key_bytes) # type: ignore[union-attr] + sha.update(value_bytes) # type: ignore[union-attr] + n_rows += 1 + n_bytes += len(key_bytes) + len(value_bytes) + + if writer is not None: + yield _finalize() diff --git a/py-sdk/mnemosyne_producer/pyproject.toml b/py-sdk/mnemosyne_producer/pyproject.toml new file mode 100644 index 00000000..8271c60f --- /dev/null +++ b/py-sdk/mnemosyne_producer/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "mnemosyne_producer" +version = "0.1.0" +description = "Mnemosyne WORM platform — Producer: Spark batch job that shards a parquet dataset by crc32(key) and emits per-shard RocksDB SST files." +requires-python = ">=3.9" +dependencies = [ + "pyspark>=3.4", + "protobuf>=4.21", + "rocksdict>=0.3", + "bharatml_commons", +] + +[project.scripts] +mnemosyne-producer = "mnemosyne_producer.job:main" + +[tool.setuptools.packages.find] +where = ["."] +include = ["mnemosyne_producer*"]