diff --git a/docs/extending/index.md b/docs/extending/index.md index 322c5047..ee82fc8d 100644 --- a/docs/extending/index.md +++ b/docs/extending/index.md @@ -13,7 +13,7 @@ both — pick the layer that matches what you're doing. | Reject SQL or YAML that's invalid in your environment before it deploys. | A `Validator` + `ValidatorProvider`. See [Validators](validators.md). | | Pull configuration values from somewhere other than `hoptimator-configmap`. | A `ConfigProvider`. See [Config providers](config-providers.md). | | Build a dependency graph from some backing store (e.g. K8s). | A `GraphProvider`. The K8s-backed default ships in `hoptimator-k8s`. | -| Render the dependency graph in a format other than Mermaid (DOT, JSON, an interactive web view, …). | A `GraphRenderer`. The Mermaid default ships in `hoptimator-graph`. | +| Render the dependency graph in a format other than the ones shipped (DOT, an interactive web view, …). | A `GraphRenderer`. Mermaid and JSON renderers ship in `hoptimator-graph`. | | Customize what gets deployed for an existing system. | Just a `TableTemplate` or `JobTemplate` — no Java needed. See [Templates and configuration](../kubernetes/templates.md). | ## How extensions are loaded @@ -77,11 +77,11 @@ The `!graph` CLI command (see goes through two SPIs: `GraphProvider` builds the typed `PipelineGraph` from some backing store, and `GraphRenderer` serializes it to a string. The bundled defaults are a K8s-backed -`K8sGraphProvider` (in `hoptimator-k8s`) and a Mermaid `MermaidRenderer` -(in `hoptimator-graph`). +`K8sGraphProvider` (in `hoptimator-k8s`) plus `MermaidRenderer` and +`JsonGraphRenderer` (in `hoptimator-graph`). Add a `GraphRenderer` to support a new output format (e.g. DOT for -graphviz, a JSON shape for a web UI). Add a `GraphProvider` if the +graphviz, an interactive web view). Add a `GraphProvider` if the pipeline state lives somewhere other than Kubernetes — the K8s implementation is the reference. Both register via `META-INF/services` like every other SPI here. diff --git a/docs/user-guide/sql-cli.md b/docs/user-guide/sql-cli.md index 1eb39246..7c9b76d5 100644 --- a/docs/user-guide/sql-cli.md +++ b/docs/user-guide/sql-cli.md @@ -44,7 +44,7 @@ commands to inspect plans, pipelines, and the deployed graph. | `!resolve` | Print the schema and source/sink connector configs Hoptimator would use for a table. | | `!pipeline` | Print the auto-generated pipeline SQL for a SELECT or CREATE MATERIALIZED VIEW statement. | | `!specify` | Print every Kubernetes spec the statement would deploy. The dry-run for `CREATE MATERIALIZED VIEW`. | -| `!graph` | Render the deployed dependency graph rooted at an identifier as a Mermaid diagram. | +| `!graph` | Render the deployed dependency graph rooted at an identifier (Mermaid by default; `--format json` for JSON). | `!resolve`, `!pipeline`, `!specify`, and `!graph` do not modify any state. Use them to sanity-check a plan before you let the JDBC driver actually @@ -117,7 +117,7 @@ If you'd `kubectl apply` the output, you'd get the same result as actually running the `CREATE MATERIALIZED VIEW`. This is the safest way to review what a statement will do before you run it. -### `!graph [--depth N]` +### `!graph [--depth N] [--format mermaid|json]` ```sql 0: Hoptimator> !graph ADS.AUDIENCE @@ -168,9 +168,19 @@ Identifier resolution runs against Calcite's catalog, so the same names you use graphs are intentionally single-hop ("what this view does," not the full upstream chain). For the chain, run `!graph` on a source identifier. -Rendering backends are pluggable. Mermaid is the default and the only one -shipped today; additional renderers can register via the -`GraphRenderer` SPI — see [Extending Hoptimator](../extending/index.md). +Rendering backends are pluggable and selected with `--format` (default +`mermaid`). A `json` renderer also ships, emitting the same graph as a +structured JSON document for programmatic consumers: + +```sql +0: Hoptimator> !graph ADS.AUDIENCE --format json +{"root":"view:ADS.AUDIENCE","orientation":"LR","nodes":[...],"edges":[...]} +``` + +Additional renderers can register via the `GraphRenderer` SPI — see +[Extending Hoptimator](../extending/index.md). `--format` accepts any +registered renderer's format identifier (case-insensitive); an unknown +format is rejected with the list of available formats. ## Running SQL diff --git a/hoptimator-cli/src/main/java/sqlline/HoptimatorAppConfig.java b/hoptimator-cli/src/main/java/sqlline/HoptimatorAppConfig.java index df242d1b..e7179978 100644 --- a/hoptimator-cli/src/main/java/sqlline/HoptimatorAppConfig.java +++ b/hoptimator-cli/src/main/java/sqlline/HoptimatorAppConfig.java @@ -307,7 +307,8 @@ public boolean echoToFile() { * !graph <schema>.<table> // 2-level identifier (case-sensitive) * !graph <catalog>.<schema>.<table> // 3-level identifier (case-sensitive) * - * Optional flag: {@code --depth N} (default 2). + * Optional flags: {@code --depth N} (default 2) and {@code --format } + * (default {@code mermaid}; any registered {@link GraphService} renderer, e.g. {@code json}). */ static final class GraphCommandHandler implements CommandHandler { @@ -329,7 +330,8 @@ public List getNames() { @Override public String getHelpText() { - return "Render a Mermaid pipeline graph for a view, logical table, or physical resource."; + return "Render a pipeline graph for a view, logical table, or physical resource " + + "(Mermaid by default; --format json for JSON)."; } @Override @@ -350,12 +352,14 @@ public void execute(String line, DispatchCallback dispatchCallback) { String[] parts = line.trim().split("\\s+"); // parts[0] = "!graph" (or "graph"); identifier mandatory. if (parts.length < 2) { - sqlline.error("Usage: !graph [--depth N]"); + sqlline.error("Usage: !graph [--depth N] [--format mermaid|json]"); dispatchCallback.setToFailure(); return; } String identifier = parts[1]; int depth = 2; + // Default to Mermaid — preserves long-standing CLI behavior and the thorough Mermaid tests. + String format = MermaidRenderer.FORMAT; // Walk through the remaining tokens. Flags consume their value (i += 1); anything else is // an unknown positional and surfaces as an error rather than getting silently dropped. int i = 2; @@ -374,9 +378,23 @@ public void execute(String line, DispatchCallback dispatchCallback) { return; } i += 2; + } else if ("--format".equals(parts[i])) { + if (i + 1 >= parts.length) { + sqlline.error("--format requires a value (e.g. mermaid, json)"); + dispatchCallback.setToFailure(); + return; + } + format = parts[i + 1]; + if (!isSupportedFormat(format, GraphService.availableFormats())) { + sqlline.error("Unsupported graph format: '" + format + "'. Available formats: " + + GraphService.availableFormats()); + dispatchCallback.setToFailure(); + return; + } + i += 2; } else { sqlline.error("Unknown argument to !graph: " + parts[i] - + ". Usage: !graph [--depth N]"); + + ". Usage: !graph [--depth N] [--format mermaid|json]"); dispatchCallback.setToFailure(); return; } @@ -385,12 +403,15 @@ public void execute(String line, DispatchCallback dispatchCallback) { HoptimatorConnection conn = (HoptimatorConnection) sqlline.getConnection(); try { PipelineGraph graph = GraphService.buildGraph(identifier, depth, conn); - sqlline.output(GraphService.render(graph, MermaidRenderer.FORMAT)); + sqlline.output(GraphService.render(graph, format)); // Resource targets root at an External node. A degenerate (root-only) Resource graph at // depth >= 1 means the label-selector found nothing — the identifier resolved to a real // schema, but no pipeline references it. Suppress the warning at depth <= 0 since the - // root-only output is the depth bound's effect, not absence of pipelines. - if (depth >= 1 && graph.root() instanceof GraphNode.External && isDegenerate(graph)) { + // root-only output is the depth bound's effect, not absence of pipelines. The warning is + // emitted as a Mermaid comment, so only append it for Mermaid output (it would corrupt + // JSON or other structured formats). + if (MermaidRenderer.FORMAT.equalsIgnoreCase(format) + && depth >= 1 && graph.root() instanceof GraphNode.External && isDegenerate(graph)) { sqlline.output(degenerateGraphWarning()); } } catch (SQLException e) { @@ -405,6 +426,15 @@ static boolean isDegenerate(PipelineGraph graph) { return graph.nodes().size() == 1 && graph.edges().isEmpty(); } + /** + * Whether {@code format} is one of the {@code availableFormats} registered with + * {@link GraphService}, compared case-insensitively. Extracted as a pure helper so the + * {@code --format} validation is unit-testable without a live sqlline shell. + */ + static boolean isSupportedFormat(String format, List availableFormats) { + return availableFormats.stream().anyMatch(f -> f.equalsIgnoreCase(format)); + } + /** Mermaid comment line warning that the resource may not exist. Comment syntax keeps the * output safe to pipe into a renderer. */ static String degenerateGraphWarning() { diff --git a/hoptimator-cli/src/test/java/sqlline/GraphCommandHandlerTest.java b/hoptimator-cli/src/test/java/sqlline/GraphCommandHandlerTest.java index f52889f8..bdc6c039 100644 --- a/hoptimator-cli/src/test/java/sqlline/GraphCommandHandlerTest.java +++ b/hoptimator-cli/src/test/java/sqlline/GraphCommandHandlerTest.java @@ -57,6 +57,27 @@ void degenerateGraphWarningStartsWithMermaidCommentSyntax() { "warning text should be self-evident: " + warning); } + @Test + void isSupportedFormatMatchesCaseInsensitively() { + List available = List.of("mermaid", "json"); + + assertTrue(HoptimatorAppConfig.GraphCommandHandler.isSupportedFormat("mermaid", available)); + assertTrue(HoptimatorAppConfig.GraphCommandHandler.isSupportedFormat("json", available), + "json renderer should be accepted when registered"); + assertTrue(HoptimatorAppConfig.GraphCommandHandler.isSupportedFormat("JSON", available), + "format comparison must be case-insensitive"); + } + + @Test + void isSupportedFormatRejectsUnregisteredFormat() { + List available = List.of("mermaid", "json"); + + assertFalse(HoptimatorAppConfig.GraphCommandHandler.isSupportedFormat("dot", available), + "an unregistered format must be rejected so the CLI can surface a usage error"); + assertFalse(HoptimatorAppConfig.GraphCommandHandler.isSupportedFormat("json", List.of("mermaid")), + "json must be rejected when only the Mermaid renderer is on the classpath"); + } + private static Set singleton(GraphNode n) { Set set = new LinkedHashSet<>(); set.add(n); diff --git a/hoptimator-graph/build.gradle b/hoptimator-graph/build.gradle index a81af0e7..e1aa9015 100644 --- a/hoptimator-graph/build.gradle +++ b/hoptimator-graph/build.gradle @@ -6,6 +6,7 @@ plugins { dependencies { implementation project(':hoptimator-api') implementation libs.cron.utils + implementation libs.jackson.databind } publishing { diff --git a/hoptimator-graph/src/main/java/com/linkedin/hoptimator/graph/json/JsonGraphRenderer.java b/hoptimator-graph/src/main/java/com/linkedin/hoptimator/graph/json/JsonGraphRenderer.java new file mode 100644 index 00000000..00438fa5 --- /dev/null +++ b/hoptimator-graph/src/main/java/com/linkedin/hoptimator/graph/json/JsonGraphRenderer.java @@ -0,0 +1,124 @@ +package com.linkedin.hoptimator.graph.json; + +import java.util.Map; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import com.linkedin.hoptimator.graph.GraphEdge; +import com.linkedin.hoptimator.graph.GraphNode; +import com.linkedin.hoptimator.graph.GraphRenderer; +import com.linkedin.hoptimator.graph.PipelineGraph; + + +/** + * {@link GraphRenderer} that serializes a {@link PipelineGraph} as stable JSON. + */ +public final class JsonGraphRenderer implements GraphRenderer { + + public static final String FORMAT = "json"; + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Override + public String format() { + return FORMAT; + } + + @Override + public String render(PipelineGraph graph) { + ObjectNode root = MAPPER.createObjectNode(); + root.put("root", graph.root().id()); + root.put("orientation", orientation(graph)); + + ArrayNode nodes = root.putArray("nodes"); + for (GraphNode node : graph.nodes()) { + nodes.add(renderNode(node)); + } + + ArrayNode edges = root.putArray("edges"); + for (GraphEdge edge : graph.edges()) { + edges.add(renderEdge(edge)); + } + + try { + return MAPPER.writeValueAsString(root); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Failed to render pipeline graph as JSON", e); + } + } + + private static ObjectNode renderNode(GraphNode node) { + ObjectNode json = MAPPER.createObjectNode(); + json.put("id", node.id()); + json.put("kind", node.kind().name()); + json.put("displayName", node.displayName()); + + switch (node.kind()) { + case PIPELINE: { + GraphNode.Pipeline pipeline = (GraphNode.Pipeline) node; + json.put("name", pipeline.name()); + putIfPresent(json, "jobKind", pipeline.jobKind()); + putIfPresent(json, "engine", pipeline.engine()); + putIfPresent(json, "executionMode", pipeline.executionMode()); + break; + } + case VIEW: { + GraphNode.View view = (GraphNode.View) node; + json.put("name", view.name()); + json.put("materialized", view.materialized()); + break; + } + case LOGICAL_TABLE: { + GraphNode.LogicalTable logicalTable = (GraphNode.LogicalTable) node; + json.put("name", logicalTable.name()); + ObjectNode tiers = json.putObject("tiers"); + for (Map.Entry tier : logicalTable.tiers().entrySet()) { + tiers.put(tier.getKey(), tier.getValue()); + } + break; + } + case TRIGGER: { + GraphNode.Trigger trigger = (GraphNode.Trigger) node; + json.put("name", trigger.name()); + putIfPresent(json, "schedule", trigger.schedule()); + json.put("paused", trigger.paused()); + putIfPresent(json, "jobTemplateName", trigger.jobTemplateName()); + putIfPresent(json, "containerName", trigger.containerName()); + break; + } + case EXTERNAL: { + GraphNode.External external = (GraphNode.External) node; + json.put("database", external.database()); + ArrayNode path = json.putArray("path"); + for (String part : external.path()) { + path.add(part); + } + break; + } + default: + throw new IllegalArgumentException("Unsupported graph node kind: " + node.kind()); + } + return json; + } + + private static ObjectNode renderEdge(GraphEdge edge) { + ObjectNode json = MAPPER.createObjectNode(); + json.put("from", edge.from().id()); + json.put("to", edge.to().id()); + json.put("type", edge.type().name()); + return json; + } + + private static void putIfPresent(ObjectNode json, String fieldName, String value) { + if (value != null) { + json.put(fieldName, value); + } + } + + private static String orientation(PipelineGraph graph) { + return graph.root().kind() == GraphNode.Kind.LOGICAL_TABLE ? "TD" : "LR"; + } +} diff --git a/hoptimator-graph/src/main/resources/META-INF/services/com.linkedin.hoptimator.graph.GraphRenderer b/hoptimator-graph/src/main/resources/META-INF/services/com.linkedin.hoptimator.graph.GraphRenderer index 43c171e3..23567208 100644 --- a/hoptimator-graph/src/main/resources/META-INF/services/com.linkedin.hoptimator.graph.GraphRenderer +++ b/hoptimator-graph/src/main/resources/META-INF/services/com.linkedin.hoptimator.graph.GraphRenderer @@ -1 +1,2 @@ com.linkedin.hoptimator.graph.mermaid.MermaidRenderer +com.linkedin.hoptimator.graph.json.JsonGraphRenderer diff --git a/hoptimator-graph/src/test/java/com/linkedin/hoptimator/graph/json/JsonGraphRendererTest.java b/hoptimator-graph/src/test/java/com/linkedin/hoptimator/graph/json/JsonGraphRendererTest.java new file mode 100644 index 00000000..d6884bdd --- /dev/null +++ b/hoptimator-graph/src/test/java/com/linkedin/hoptimator/graph/json/JsonGraphRendererTest.java @@ -0,0 +1,169 @@ +package com.linkedin.hoptimator.graph.json; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import com.linkedin.hoptimator.graph.GraphEdge; +import com.linkedin.hoptimator.graph.GraphNode; +import com.linkedin.hoptimator.graph.PipelineGraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/** + * Unit tests for {@link JsonGraphRenderer}. These exercise the JSON contract in isolation + * (hand-built {@link PipelineGraph} fixtures, no K8s involvement). + */ +class JsonGraphRendererTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Test + void rendersAllNodeKindsAndIncludesOwnerEdges() throws Exception { + GraphNode.View root = new GraphNode.View("audience", true); + GraphNode.Pipeline pipeline = new GraphNode.Pipeline("audience-pipe", + "FlinkDeployment", "Flink", "Streaming"); + GraphNode.Trigger trigger = new GraphNode.Trigger("audience-trigger", + "0 */6 * * *", true, "retl-job-template", "main"); + GraphNode.External external = new GraphNode.External("kafka-db", + Arrays.asList("KAFKA", "events")); + Map tiers = new LinkedHashMap<>(); + tiers.put("nearline", "kafka-db"); + GraphNode.LogicalTable logicalTable = new GraphNode.LogicalTable("audience-lt", tiers); + + Set nodes = setOf(root, pipeline, trigger, external, logicalTable); + Set edges = setOf( + new GraphEdge(root, pipeline, GraphEdge.Type.OWNER_OF), + new GraphEdge(trigger, pipeline, GraphEdge.Type.TRIGGERS), + new GraphEdge(external, pipeline, GraphEdge.Type.DEPENDS_ON_SOURCE)); + + JsonNode json = render(new PipelineGraph(root, nodes, edges)); + + assertEquals("json", new JsonGraphRenderer().format()); + assertEquals(root.id(), json.get("root").asText()); + assertEquals("LR", json.get("orientation").asText()); + assertEquals(root.id(), json.get("nodes").get(0).get("id").asText()); + assertEquals("VIEW", node(json, root).get("kind").asText()); + assertEquals("PIPELINE", node(json, pipeline).get("kind").asText()); + assertEquals("TRIGGER", node(json, trigger).get("kind").asText()); + assertEquals("EXTERNAL", node(json, external).get("kind").asText()); + assertEquals("LOGICAL_TABLE", node(json, logicalTable).get("kind").asText()); + + JsonNode pipelineJson = node(json, pipeline); + assertEquals("audience-pipe", pipelineJson.get("displayName").asText()); + assertEquals("audience-pipe", pipelineJson.get("name").asText()); + assertEquals("FlinkDeployment", pipelineJson.get("jobKind").asText()); + assertEquals("Flink", pipelineJson.get("engine").asText()); + assertEquals("Streaming", pipelineJson.get("executionMode").asText()); + + JsonNode triggerJson = node(json, trigger); + assertEquals("0 */6 * * *", triggerJson.get("schedule").asText()); + assertTrue(triggerJson.get("paused").asBoolean()); + assertEquals("retl-job-template", triggerJson.get("jobTemplateName").asText()); + assertEquals("main", triggerJson.get("containerName").asText()); + + JsonNode externalJson = node(json, external); + assertEquals("kafka-db", externalJson.get("database").asText()); + assertEquals("KAFKA", externalJson.get("path").get(0).asText()); + assertEquals("events", externalJson.get("path").get(1).asText()); + + JsonNode ownerEdge = edge(json, root, pipeline, GraphEdge.Type.OWNER_OF); + assertNotNull(ownerEdge, "OWNER_OF edges must be included in JSON output"); + assertEquals("TRIGGERS", json.get("edges").get(1).get("type").asText()); + } + + @Test + void logicalTableRootUsesTdOrientationAndPreservesTierOrder() throws Exception { + Map tiers = new LinkedHashMap<>(); + tiers.put("nearline", "kafka-db"); + tiers.put("online", "venice-db"); + tiers.put("offline", "hdfs-db"); + GraphNode.LogicalTable root = new GraphNode.LogicalTable("foo", tiers); + Set nodes = setOf(root); + Set edges = setOf(); + + JsonNode json = render(new PipelineGraph(root, nodes, edges)); + + assertEquals("TD", json.get("orientation").asText()); + JsonNode tiersJson = node(json, root).get("tiers"); + assertEquals("kafka-db", tiersJson.get("nearline").asText()); + assertEquals("venice-db", tiersJson.get("online").asText()); + assertEquals("hdfs-db", tiersJson.get("offline").asText()); + + Iterator fieldNames = tiersJson.fieldNames(); + assertEquals("nearline", fieldNames.next()); + assertEquals("online", fieldNames.next()); + assertEquals("offline", fieldNames.next()); + assertFalse(fieldNames.hasNext()); + } + + @Test + void nullableFieldsAreOmittedButBooleansAreAlwaysIncluded() throws Exception { + GraphNode.Pipeline pipeline = new GraphNode.Pipeline("p1", null, null, null); + GraphNode.Trigger trigger = new GraphNode.Trigger("t1", null, false, null, null); + GraphNode.View view = new GraphNode.View("v1", false); + Set nodes = setOf(pipeline, trigger, view); + Set edges = setOf(); + + JsonNode json = render(new PipelineGraph(pipeline, nodes, edges)); + + assertEquals("LR", json.get("orientation").asText()); + JsonNode pipelineJson = node(json, pipeline); + assertFalse(pipelineJson.has("jobKind")); + assertFalse(pipelineJson.has("engine")); + assertFalse(pipelineJson.has("executionMode")); + + JsonNode triggerJson = node(json, trigger); + assertFalse(triggerJson.has("schedule")); + assertFalse(triggerJson.get("paused").asBoolean()); + assertFalse(triggerJson.has("jobTemplateName")); + assertFalse(triggerJson.has("containerName")); + + JsonNode viewJson = node(json, view); + assertTrue(viewJson.has("materialized")); + assertFalse(viewJson.get("materialized").asBoolean()); + } + + private static JsonNode render(PipelineGraph graph) throws Exception { + return MAPPER.readTree(new JsonGraphRenderer().render(graph)); + } + + private static JsonNode node(JsonNode graph, GraphNode node) { + for (JsonNode n : graph.get("nodes")) { + if (node.id().equals(n.get("id").asText())) { + return n; + } + } + throw new AssertionError("Missing node " + node.id() + " in " + graph); + } + + private static JsonNode edge(JsonNode graph, GraphNode from, GraphNode to, GraphEdge.Type type) { + for (JsonNode edge : graph.get("edges")) { + if (from.id().equals(edge.get("from").asText()) + && to.id().equals(edge.get("to").asText()) + && type.name().equals(edge.get("type").asText())) { + return edge; + } + } + return null; + } + + @SafeVarargs + private static Set setOf(T... items) { + Set set = new LinkedHashSet<>(); + Collections.addAll(set, items); + return set; + } +}