diff --git a/Semantic Model AI Metadata/Edit Semantic Model AI Instructions.csx b/Semantic Model AI Metadata/Edit Semantic Model AI Instructions.csx new file mode 100644 index 0000000..187046b --- /dev/null +++ b/Semantic Model AI Metadata/Edit Semantic Model AI Instructions.csx @@ -0,0 +1,395 @@ +#r "System.Drawing" + +/* + * Title: Edit semantic model AI instructions + * + * Author: Kurt Buhler + * + * This Tabular Editor 3 Desktop script opens a GUI editor for semantic model + * AI instructions stored in culture linguistic metadata CustomInstructions. + * It uses the connected model, defaults to the en-US culture, and does not + * require a selected object. + * + * Provided as-is. This script edits culture linguistic metadata directly and is + * not an official supported Tabular Editor product feature. + */ + +// TE3 Desktop GUI editor for semantic model AI instructions. +// Uses Model.Cultures["en-US"].Content -> CustomInstructions. + +using System; +using System.Drawing; +using System.Linq; +using System.Reflection; +using System.Windows.Forms; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using TabularEditor.TOMWrapper; + +try +{ + AiInstructionsEditor.Run(Model); +} +catch (Exception ex) +{ + MessageBox.Show(AiInstructionsEditor.RootMessage(ex), "Semantic Model AI Instructions"); +} + +public static class AiInstructionsEditor +{ + private const string DefaultCultureName = "en-US"; + private const int InstructionsLimit = 10000; + + public static void Run(TabularEditor.TOMWrapper.Model model) + { + ScriptHelper.WaitFormVisible = false; + + if (model == null) + { + MessageBox.Show("Open or connect to a model before running this script.", "Semantic Model AI Instructions"); + return; + } + + string startupWarning; + var culture = EnsureCulture(model, DefaultCultureName, out startupWarning); + if (culture == null) + { + MessageBox.Show("Could not find or create the en-US culture.", "Semantic Model AI Instructions"); + return; + } + + using (var form = new Form()) + using (var editor = ScriptTextEditor.Create(true)) + { + form.Text = "Semantic Model AI Instructions"; + form.StartPosition = FormStartPosition.CenterScreen; + form.AutoScaleMode = AutoScaleMode.Dpi; + form.Width = 980; + form.Height = 760; + form.MinimumSize = new Size(760, 520); + + var font = new Font("Segoe UI", 9F); + + var layout = new TableLayoutPanel + { + Dock = DockStyle.Fill, + ColumnCount = 1, + RowCount = 3, + Padding = new Padding(10) + }; + layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 34F)); + layout.RowStyles.Add(new RowStyle(SizeType.Percent, 100F)); + layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 58F)); + + var header = new Label + { + Dock = DockStyle.Fill, + AutoEllipsis = true, + TextAlign = ContentAlignment.MiddleLeft, + Font = font, + Text = "Model: " + model.Name + " Culture: " + culture.Name + }; + + var editorPanel = new Panel + { + Dock = DockStyle.Fill, + BorderStyle = BorderStyle.FixedSingle, + Padding = new Padding(0) + }; + editorPanel.Controls.Add(editor.Control); + + var bottom = new TableLayoutPanel + { + Dock = DockStyle.Fill, + ColumnCount = 2, + RowCount = 1, + Margin = new Padding(0), + Padding = new Padding(0, 8, 0, 4) + }; + bottom.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100F)); + bottom.ColumnStyles.Add(new ColumnStyle(SizeType.AutoSize)); + + var status = new Label + { + Dock = DockStyle.Fill, + AutoEllipsis = true, + TextAlign = ContentAlignment.MiddleLeft, + Font = font, + Margin = new Padding(0, 0, 8, 0) + }; + + var wrapButton = NewFooterButton("Wrap", font); + var reloadButton = NewFooterButton("Reload", font); + var saveButton = NewFooterButton("Save", font); + var closeButton = NewFooterButton("Close", font); + closeButton.DialogResult = DialogResult.Cancel; + + var buttonStrip = new FlowLayoutPanel + { + Dock = DockStyle.Fill, + AutoSize = true, + AutoSizeMode = AutoSizeMode.GrowAndShrink, + FlowDirection = FlowDirection.LeftToRight, + WrapContents = false, + Margin = new Padding(0), + Padding = new Padding(0) + }; + buttonStrip.Controls.Add(wrapButton); + buttonStrip.Controls.Add(reloadButton); + buttonStrip.Controls.Add(saveButton); + buttonStrip.Controls.Add(closeButton); + + bottom.Controls.Add(status, 0, 0); + bottom.Controls.Add(buttonStrip, 1, 0); + + layout.Controls.Add(header, 0, 0); + layout.Controls.Add(editorPanel, 0, 1); + layout.Controls.Add(bottom, 0, 2); + form.Controls.Add(layout); + form.CancelButton = closeButton; + + Action refreshStatus = () => + { + var count = NormalizeForStorage(editor.Text).Length; + status.Text = count + " / " + InstructionsLimit + " characters" + + (string.IsNullOrWhiteSpace(startupWarning) ? "" : " " + startupWarning); + status.ForeColor = count > InstructionsLimit ? Color.Firebrick : + string.IsNullOrWhiteSpace(startupWarning) ? SystemColors.ControlText : Color.DarkGoldenrod; + saveButton.Enabled = count <= InstructionsLimit; + }; + + Action load = () => + { + try + { + editor.Text = NormalizeForEditor(GetInstructions(culture)); + editor.SelectStart(); + refreshStatus(); + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + + editor.TextChanged += (sender, args) => refreshStatus(); + wrapButton.Click += (sender, args) => editor.WordWrap = !editor.WordWrap; + reloadButton.Click += (sender, args) => load(); + saveButton.Click += (sender, args) => + { + try + { + var text = NormalizeForStorage(editor.Text); + if (text.Length > InstructionsLimit) + { + status.Text = "AI instructions must be 10000 characters or fewer."; + status.ForeColor = Color.Firebrick; + return; + } + + SetInstructions(culture, text); + status.Text = "Saved to " + culture.Name + ". Save the model to persist."; + status.ForeColor = Color.ForestGreen; + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + closeButton.Click += (sender, args) => form.Close(); + + load(); + form.Shown += (sender, args) => editor.Focus(); + form.ShowDialog(); + } + } + + private static Culture EnsureCulture(TabularEditor.TOMWrapper.Model model, string cultureName, out string warning) + { + warning = null; + + if (model.Cultures.Contains(cultureName)) return model.Cultures[cultureName]; + + try + { + return model.AddTranslation(cultureName); + } + catch + { + // Power BI Desktop-connected models may block AddTranslation. Fall back to TE's import helper. + } + + try + { + if (TryImportEmptyCulture(model, cultureName) && model.Cultures.Contains(cultureName)) + { + warning = "Created " + cultureName + " culture."; + return model.Cultures[cultureName]; + } + } + catch + { + // Fall through to an existing culture or a controlled message. + } + + var fallback = model.Cultures.FirstOrDefault(c => !string.IsNullOrWhiteSpace(c.Content)) + ?? model.Cultures.FirstOrDefault(); + if (fallback != null) + { + warning = "Could not create " + cultureName + "; using " + fallback.Name + "."; + return fallback; + } + + return null; + } + + private static Button NewFooterButton(string text, Font font) + { + return new Button + { + Text = text, + Dock = DockStyle.Fill, + Font = font, + AutoSize = true, + AutoSizeMode = AutoSizeMode.GrowAndShrink, + Margin = new Padding(6, 2, 0, 2), + MinimumSize = new Size(96, 32), + Padding = new Padding(12, 0, 12, 0), + TextAlign = ContentAlignment.MiddleCenter, + UseVisualStyleBackColor = true + }; + } + + private static bool TryImportEmptyCulture(TabularEditor.TOMWrapper.Model model, string cultureName) + { + var helperType = typeof(TabularEditor.TOMWrapper.Model).Assembly.GetType("TabularEditor.TOMWrapper.TabularCultureHelper"); + if (helperType == null) return false; + + var method = helperType.GetMethod("ImportCulture", BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); + if (method == null) return false; + + var cultureJson = new JObject { ["name"] = cultureName }; + var result = method.Invoke(null, new object[] { cultureJson, model, false, true }); + return result is bool ok && ok; + } + + private static string GetInstructions(Culture culture) + { + var payload = GetPayload(culture, false); + return (string)payload["CustomInstructions"] ?? ""; + } + + private static void SetInstructions(Culture culture, string instructions) + { + var payload = GetPayload(culture, true); + payload["CustomInstructions"] = instructions ?? ""; + SavePayload(culture, payload); + } + + private static JObject GetPayload(Culture culture, bool create) + { + if (!string.IsNullOrWhiteSpace(culture.Content)) + { + return JObject.Parse(culture.Content); + } + + if (!create) return new JObject(); + + return new JObject + { + ["Version"] = "4.2.0", + ["Language"] = culture.Name, + ["Entities"] = new JObject(), + ["Agents"] = new JObject + { + ["Internal"] = new JObject { ["Version"] = "1.1.0" } + } + }; + } + + private static void SavePayload(Culture culture, JObject payload) + { + culture.Content = payload.ToString(Formatting.Indented); + } + + private static string NormalizeForEditor(string text) + { + return (text ?? "").Replace("\r\n", "\n").Replace("\r", "\n").Replace("\n", Environment.NewLine); + } + + private static string NormalizeForStorage(string text) + { + return (text ?? "").Replace("\r\n", "\n").Replace("\r", "\n"); + } + + public static string RootMessage(Exception ex) + { + if (ex == null) return ""; + while (ex.InnerException != null) ex = ex.InnerException; + return ex.Message; + } +} + +public sealed class ScriptTextEditor : IDisposable +{ + private readonly TextBox textBox; + + public Control Control { get; private set; } + public event EventHandler TextChanged; + + private ScriptTextEditor(TextBox textBox) + { + this.textBox = textBox; + Control = textBox; + Control.TextChanged += (sender, args) => TextChanged?.Invoke(this, EventArgs.Empty); + } + + public static ScriptTextEditor Create(bool wordWrap) + { + var textBox = new TextBox + { + Dock = DockStyle.Fill, + Multiline = true, + ScrollBars = ScrollBars.Both, + WordWrap = wordWrap, + AcceptsReturn = true, + AcceptsTab = true, + Font = new Font("Consolas", 10F), + BorderStyle = BorderStyle.None + }; + return new ScriptTextEditor(textBox); + } + + public string Text + { + get { return Control.Text ?? ""; } + set { Control.Text = value ?? ""; } + } + + public bool WordWrap + { + get { return textBox.WordWrap; } + set + { + textBox.WordWrap = value; + } + } + + public void Focus() + { + Control.Focus(); + } + + public void SelectStart() + { + textBox.SelectionStart = 0; + textBox.SelectionLength = 0; + } + + public void Dispose() + { + Control?.Dispose(); + } +} diff --git a/Semantic Model AI Metadata/Edit Semantic Model AI Schema.csx b/Semantic Model AI Metadata/Edit Semantic Model AI Schema.csx new file mode 100644 index 0000000..7493993 --- /dev/null +++ b/Semantic Model AI Metadata/Edit Semantic Model AI Schema.csx @@ -0,0 +1,1130 @@ +#r "System.Drawing" + +/* + * Title: Edit semantic model AI schema + * + * Author: Kurt Buhler + * + * This Tabular Editor 3 Desktop script opens a GUI editor for semantic model + * AI schema stored in culture linguistic metadata Entities. The object tree + * follows the perspective-editor interaction model and includes a JSON tab + * for exact import/export roundtrips. + * + * Provided as-is. This script edits culture linguistic metadata directly and is + * not an official supported Tabular Editor product feature. + */ + +// TE3 Desktop GUI editor for semantic model AI schema. +// Uses Model.Cultures["en-US"].Content -> Entities. + +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Linq; +using System.Reflection; +using System.Text.RegularExpressions; +using System.Windows.Forms; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using TabularEditor.TOMWrapper; + +try +{ + AiSchemaEditor.Run(Model); +} +catch (Exception ex) +{ + MessageBox.Show(AiSchemaEditor.RootMessage(ex), "Semantic Model AI Schema"); +} + +public static class AiSchemaEditor +{ + private const string DefaultCultureName = "en-US"; + + public static void Run(TabularEditor.TOMWrapper.Model model) + { + ScriptHelper.WaitFormVisible = false; + + if (model == null) + { + MessageBox.Show("Open or connect to a model before running this script.", "Semantic Model AI Schema"); + return; + } + + string startupWarning; + var culture = EnsureCulture(model, DefaultCultureName, out startupWarning); + if (culture == null) + { + MessageBox.Show("Could not find or create the en-US culture.", "Semantic Model AI Schema"); + return; + } + + using (var form = new Form()) + using (var jsonEditor = ScriptTextEditor.Create(false)) + using (var stateImages = BuildStateImages()) + { + form.Text = "Semantic Model AI Schema"; + form.StartPosition = FormStartPosition.CenterScreen; + form.AutoScaleMode = AutoScaleMode.Dpi; + form.Width = 1040; + form.Height = 780; + form.MinimumSize = new Size(820, 560); + + var font = new Font("Segoe UI", 9F); + + var layout = new TableLayoutPanel + { + Dock = DockStyle.Fill, + ColumnCount = 1, + RowCount = 3, + Padding = new Padding(10) + }; + layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 34F)); + layout.RowStyles.Add(new RowStyle(SizeType.Percent, 100F)); + layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 58F)); + + var header = new Label + { + Dock = DockStyle.Fill, + AutoEllipsis = true, + TextAlign = ContentAlignment.MiddleLeft, + Font = font, + Text = "Model: " + model.Name + " Culture: " + culture.Name + }; + + var tabs = new TabControl + { + Dock = DockStyle.Fill, + Font = font + }; + var treePage = new TabPage("Objects"); + var jsonPage = new TabPage("JSON"); + tabs.TabPages.Add(treePage); + tabs.TabPages.Add(jsonPage); + + var tree = new TreeView + { + Dock = DockStyle.Fill, + BorderStyle = BorderStyle.None, + Font = font, + HideSelection = false, + ShowLines = true, + ShowPlusMinus = true, + ShowRootLines = true, + StateImageList = stateImages + }; + + var treePanel = new Panel + { + Dock = DockStyle.Fill, + BorderStyle = BorderStyle.FixedSingle + }; + treePanel.Controls.Add(tree); + + var toolbar = new FlowLayoutPanel + { + Dock = DockStyle.Top, + Height = 44, + FlowDirection = FlowDirection.LeftToRight, + WrapContents = false, + Padding = new Padding(0, 6, 0, 4) + }; + var showHidden = new CheckBox { Text = "Show hidden", Checked = true, AutoSize = true, Font = font, Padding = new Padding(0, 7, 12, 0) }; + var checkAllButton = NewToolbarButton("Check all", font); + var clearButton = NewToolbarButton("Clear", font); + var expandButton = NewToolbarButton("Expand", font); + var collapseButton = NewToolbarButton("Collapse", font); + toolbar.Controls.Add(showHidden); + toolbar.Controls.Add(checkAllButton); + toolbar.Controls.Add(clearButton); + toolbar.Controls.Add(expandButton); + toolbar.Controls.Add(collapseButton); + + var treeLayout = new Panel { Dock = DockStyle.Fill, Padding = new Padding(0) }; + treeLayout.Controls.Add(treePanel); + treeLayout.Controls.Add(toolbar); + treePage.Controls.Add(treeLayout); + + var jsonPanel = new Panel + { + Dock = DockStyle.Fill, + BorderStyle = BorderStyle.FixedSingle, + Padding = new Padding(0) + }; + jsonPanel.Controls.Add(jsonEditor.Control); + jsonPage.Controls.Add(jsonPanel); + + var bottom = new TableLayoutPanel + { + Dock = DockStyle.Fill, + ColumnCount = 2, + RowCount = 1, + Margin = new Padding(0), + Padding = new Padding(0, 8, 0, 4) + }; + bottom.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100F)); + bottom.ColumnStyles.Add(new ColumnStyle(SizeType.AutoSize)); + + var status = new Label + { + Dock = DockStyle.Fill, + AutoEllipsis = true, + TextAlign = ContentAlignment.MiddleLeft, + Font = font, + ForeColor = string.IsNullOrWhiteSpace(startupWarning) ? SystemColors.ControlText : Color.DarkGoldenrod, + Margin = new Padding(0, 0, 8, 0) + }; + + var reloadButton = NewFooterButton("Reload", font); + var formatButton = NewFooterButton("Format JSON", font); + var updateJsonButton = NewFooterButton("Update JSON", font); + var saveButton = NewFooterButton("Save", font); + var closeButton = NewFooterButton("Close", font); + closeButton.DialogResult = DialogResult.Cancel; + + var buttonStrip = new FlowLayoutPanel + { + Dock = DockStyle.Fill, + AutoSize = true, + AutoSizeMode = AutoSizeMode.GrowAndShrink, + FlowDirection = FlowDirection.LeftToRight, + WrapContents = false, + Margin = new Padding(0), + Padding = new Padding(0) + }; + buttonStrip.Controls.Add(reloadButton); + buttonStrip.Controls.Add(formatButton); + buttonStrip.Controls.Add(updateJsonButton); + buttonStrip.Controls.Add(saveButton); + buttonStrip.Controls.Add(closeButton); + + bottom.Controls.Add(status, 0, 0); + bottom.Controls.Add(buttonStrip, 1, 0); + + layout.Controls.Add(header, 0, 0); + layout.Controls.Add(tabs, 0, 1); + layout.Controls.Add(bottom, 0, 2); + form.Controls.Add(layout); + form.CancelButton = closeButton; + + bool updatingTree = false; + + Action loadSchemaIntoUi = schema => + { + updatingTree = true; + try + { + BuildTree(tree, model, schema, showHidden.Checked); + jsonEditor.Text = NormalizeForEditor(schema.ToString(Formatting.Indented)); + status.Text = StatusText(tree, startupWarning); + status.ForeColor = string.IsNullOrWhiteSpace(startupWarning) ? SystemColors.ControlText : Color.DarkGoldenrod; + } + finally + { + updatingTree = false; + } + }; + + Action reload = () => + { + try + { + loadSchemaIntoUi(GetSchema(culture)); + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + + tree.NodeMouseClick += (sender, args) => + { + if (updatingTree) return; + try + { + ToggleNode(args.Node); + status.Text = StatusText(tree, startupWarning); + status.ForeColor = SystemColors.ControlText; + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + + showHidden.CheckedChanged += (sender, args) => + { + if (updatingTree) return; + try + { + var current = SchemaFromTree(tree); + loadSchemaIntoUi(current); + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + + checkAllButton.Click += (sender, args) => + { + SetAllTreeNodes(tree, CheckedState); + status.Text = StatusText(tree, startupWarning); + status.ForeColor = SystemColors.ControlText; + }; + clearButton.Click += (sender, args) => + { + SetAllTreeNodes(tree, UncheckedState); + status.Text = StatusText(tree, startupWarning); + status.ForeColor = SystemColors.ControlText; + }; + expandButton.Click += (sender, args) => tree.ExpandAll(); + collapseButton.Click += (sender, args) => tree.CollapseAll(); + + reloadButton.Click += (sender, args) => reload(); + formatButton.Click += (sender, args) => + { + try + { + jsonEditor.Text = NormalizeForEditor(JObject.Parse(jsonEditor.Text).ToString(Formatting.Indented)); + tabs.SelectedTab = jsonPage; + status.Text = "JSON formatted."; + status.ForeColor = SystemColors.ControlText; + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + updateJsonButton.Click += (sender, args) => + { + try + { + jsonEditor.Text = NormalizeForEditor(SchemaFromTree(tree).ToString(Formatting.Indented)); + tabs.SelectedTab = jsonPage; + jsonEditor.SelectStart(); + status.Text = "JSON updated from object tree."; + status.ForeColor = SystemColors.ControlText; + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + saveButton.Click += (sender, args) => + { + try + { + JObject schema; + if (tabs.SelectedTab == jsonPage) + { + schema = JObject.Parse(jsonEditor.Text); + SetSchema(culture, schema); + loadSchemaIntoUi(schema); + tabs.SelectedTab = jsonPage; + } + else + { + schema = SchemaFromTree(tree); + SetSchema(culture, schema); + jsonEditor.Text = NormalizeForEditor(schema.ToString(Formatting.Indented)); + } + + status.Text = "Saved to " + culture.Name + ". Save the model to persist."; + status.ForeColor = Color.ForestGreen; + } + catch (Exception ex) + { + status.Text = RootMessage(ex); + status.ForeColor = Color.Firebrick; + } + }; + closeButton.Click += (sender, args) => form.Close(); + + reload(); + form.ShowDialog(); + } + } + + private const int UncheckedState = 0; + private const int CheckedState = 1; + private const int MixedState = 2; + + private static void BuildTree(TreeView tree, TabularEditor.TOMWrapper.Model model, JObject schema, bool showHidden) + { + tree.BeginUpdate(); + try + { + tree.Nodes.Clear(); + var index = BuildSchemaIndex(schema); + var hasSchema = index.Count > 0; + + foreach (var table in model.Tables.OrderBy(t => t.Name)) + { + if (!showHidden && !IsVisibleObject(table)) continue; + + var tableNode = NewNode(table.Name, new SchemaNode("table", table.Name, null, null, null)); + tree.Nodes.Add(tableNode); + + foreach (var column in table.Columns.OrderBy(c => c.Name)) + { + if (!showHidden && !IsVisibleObject(column)) continue; + var node = NewNode(column.Name + " column", new SchemaNode("column", table.Name, column.Name, null, null)); + node.ForeColor = IsVisibleObject(column) ? SystemColors.WindowText : SystemColors.GrayText; + node.StateImageIndex = IncludeState(index, node.Tag as SchemaNode, hasSchema, IsVisibleObject(column)); + tableNode.Nodes.Add(node); + } + + foreach (var measure in table.Measures.OrderBy(m => m.Name)) + { + if (!showHidden && !IsVisibleObject(measure)) continue; + var node = NewNode(measure.Name + " measure", new SchemaNode("measure", table.Name, measure.Name, null, null)); + node.ForeColor = IsVisibleObject(measure) ? SystemColors.WindowText : SystemColors.GrayText; + node.StateImageIndex = IncludeState(index, node.Tag as SchemaNode, hasSchema, IsVisibleObject(measure)); + tableNode.Nodes.Add(node); + } + + foreach (var hierarchy in table.Hierarchies.OrderBy(h => h.Name)) + { + if (!showHidden && !IsVisibleObject(hierarchy)) continue; + var hierarchyNode = NewNode(hierarchy.Name + " hierarchy", new SchemaNode("hierarchy", table.Name, null, hierarchy.Name, null)); + hierarchyNode.ForeColor = IsVisibleObject(hierarchy) ? SystemColors.WindowText : SystemColors.GrayText; + hierarchyNode.StateImageIndex = IncludeState(index, hierarchyNode.Tag as SchemaNode, hasSchema, IsVisibleObject(hierarchy)); + tableNode.Nodes.Add(hierarchyNode); + + foreach (var level in hierarchy.Levels.OrderBy(l => l.Name)) + { + var levelNode = NewNode(level.Name + " level", new SchemaNode("level", table.Name, null, hierarchy.Name, level.Name)); + levelNode.StateImageIndex = IncludeState(index, levelNode.Tag as SchemaNode, hasSchema, true); + hierarchyNode.Nodes.Add(levelNode); + } + + if (hierarchyNode.Nodes.Count > 0 && !index.ContainsKey(Key(hierarchyNode.Tag as SchemaNode))) + { + hierarchyNode.StateImageIndex = AggregateState(hierarchyNode); + } + } + + tableNode.ForeColor = IsVisibleObject(table) ? SystemColors.WindowText : SystemColors.GrayText; + tableNode.StateImageIndex = index.ContainsKey(Key(tableNode.Tag as SchemaNode)) + ? index[Key(tableNode.Tag as SchemaNode)] ? CheckedState : UncheckedState + : tableNode.Nodes.Count == 0 ? (IsVisibleObject(table) || !hasSchema ? CheckedState : UncheckedState) : AggregateState(tableNode); + tableNode.Expand(); + } + } + finally + { + tree.EndUpdate(); + } + } + + private static Button NewFooterButton(string text, Font font) + { + return new Button + { + Text = text, + Dock = DockStyle.Fill, + Font = font, + AutoSize = true, + AutoSizeMode = AutoSizeMode.GrowAndShrink, + Margin = new Padding(6, 2, 0, 2), + MinimumSize = new Size(96, 32), + Padding = new Padding(12, 0, 12, 0), + TextAlign = ContentAlignment.MiddleCenter, + UseVisualStyleBackColor = true + }; + } + + private static Button NewToolbarButton(string text, Font font) + { + return new Button + { + Text = text, + AutoSize = true, + AutoSizeMode = AutoSizeMode.GrowAndShrink, + Font = font, + Margin = new Padding(6, 0, 0, 0), + MinimumSize = new Size(96, 30), + Padding = new Padding(12, 0, 12, 0), + TextAlign = ContentAlignment.MiddleCenter, + UseVisualStyleBackColor = true + }; + } + + private static TreeNode NewNode(string text, SchemaNode tag) + { + return new TreeNode(text) + { + Tag = tag, + StateImageIndex = CheckedState + }; + } + + private static bool IsVisibleObject(object obj) + { + if (obj is IHideableObject hideable) return hideable.IsVisible; + return true; + } + + private static int IncludeState(Dictionary index, SchemaNode node, bool hasSchema, bool visible) + { + var key = Key(node); + if (index.ContainsKey(key)) return index[key] ? CheckedState : UncheckedState; + return !hasSchema || visible ? CheckedState : UncheckedState; + } + + private static Dictionary BuildSchemaIndex(JObject schema) + { + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var tableEntry in CollectionEntries(schema?["tables"] ?? schema?["Tables"])) + { + var table = tableEntry.Value as JObject; + var tableName = SchemaObjectName(tableEntry.Key, tableEntry.Value); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + result[Key(new SchemaNode("table", tableName, null, null, null))] = IncludeValue(tableEntry.Value); + + foreach (var columnEntry in CollectionEntries(table?["columns"] ?? table?["Columns"])) + { + var name = SchemaObjectName(columnEntry.Key, columnEntry.Value); + if (!string.IsNullOrWhiteSpace(name)) result[Key(new SchemaNode("column", tableName, name, null, null))] = IncludeValue(columnEntry.Value); + } + + foreach (var measureEntry in CollectionEntries(table?["measures"] ?? table?["Measures"])) + { + var name = SchemaObjectName(measureEntry.Key, measureEntry.Value); + if (!string.IsNullOrWhiteSpace(name)) result[Key(new SchemaNode("measure", tableName, name, null, null))] = IncludeValue(measureEntry.Value); + } + + foreach (var hierarchyEntry in CollectionEntries(table?["hierarchies"] ?? table?["Hierarchies"])) + { + var hierarchy = hierarchyEntry.Value as JObject; + var name = SchemaObjectName(hierarchyEntry.Key, hierarchyEntry.Value); + if (string.IsNullOrWhiteSpace(name)) continue; + result[Key(new SchemaNode("hierarchy", tableName, null, name, null))] = IncludeValue(hierarchyEntry.Value); + + foreach (var levelEntry in CollectionEntries(hierarchy?["levels"] ?? hierarchy?["Levels"])) + { + var level = SchemaObjectName(levelEntry.Key, levelEntry.Value); + if (!string.IsNullOrWhiteSpace(level)) result[Key(new SchemaNode("level", tableName, null, name, level))] = IncludeValue(levelEntry.Value); + } + } + } + return result; + } + + private static string Key(SchemaNode node) + { + if (node == null) return ""; + if (node.Kind == "table") return "T|" + node.Table; + if (node.Kind == "column" || node.Kind == "measure") return "P|" + node.Table + "|" + node.Property; + if (node.Kind == "hierarchy") return "H|" + node.Table + "|" + node.Hierarchy; + if (node.Kind == "level") return "L|" + node.Table + "|" + node.Hierarchy + "|" + node.Level; + return ""; + } + + private static void ToggleNode(TreeNode node) + { + var next = node.StateImageIndex == CheckedState ? UncheckedState : CheckedState; + SetNodeAndChildren(node, next); + RefreshAncestors(node.Parent); + } + + private static void SetAllTreeNodes(TreeView tree, int state) + { + tree.BeginUpdate(); + try + { + foreach (TreeNode node in tree.Nodes) + { + SetNodeAndChildren(node, state); + } + } + finally + { + tree.EndUpdate(); + } + } + + private static void SetNodeAndChildren(TreeNode node, int state) + { + node.StateImageIndex = state; + foreach (TreeNode child in node.Nodes) + { + SetNodeAndChildren(child, state); + } + } + + private static void RefreshAncestors(TreeNode node) + { + while (node != null) + { + node.StateImageIndex = AggregateState(node); + node = node.Parent; + } + } + + private static int AggregateState(TreeNode node) + { + if (node.Nodes.Count == 0) return node.StateImageIndex == CheckedState ? CheckedState : UncheckedState; + + var checkedCount = 0; + var uncheckedCount = 0; + foreach (TreeNode child in node.Nodes) + { + if (child.StateImageIndex == CheckedState) checkedCount++; + else if (child.StateImageIndex == UncheckedState) uncheckedCount++; + else return MixedState; + } + + if (checkedCount == node.Nodes.Count) return CheckedState; + if (uncheckedCount == node.Nodes.Count) return UncheckedState; + return MixedState; + } + + private static JObject SchemaFromTree(TreeView tree) + { + var tables = new JArray(); + foreach (TreeNode tableNode in tree.Nodes) + { + var tableTag = tableNode.Tag as SchemaNode; + if (tableTag == null || tableTag.Kind != "table") continue; + + var table = new JObject + { + ["name"] = tableTag.Table, + ["include"] = tableNode.StateImageIndex != UncheckedState + }; + var columns = new JArray(); + var measures = new JArray(); + var hierarchies = new JArray(); + + foreach (TreeNode child in tableNode.Nodes) + { + var tag = child.Tag as SchemaNode; + if (tag == null) continue; + + if (tag.Kind == "column") + { + columns.Add(new JObject { ["name"] = tag.Property, ["include"] = child.StateImageIndex == CheckedState }); + } + else if (tag.Kind == "measure") + { + measures.Add(new JObject { ["name"] = tag.Property, ["include"] = child.StateImageIndex == CheckedState }); + } + else if (tag.Kind == "hierarchy") + { + var hierarchy = new JObject + { + ["name"] = tag.Hierarchy, + ["include"] = child.StateImageIndex != UncheckedState + }; + var levels = new JArray(); + foreach (TreeNode levelNode in child.Nodes) + { + var levelTag = levelNode.Tag as SchemaNode; + if (levelTag != null && levelTag.Kind == "level") + { + levels.Add(new JObject { ["name"] = levelTag.Level, ["include"] = levelNode.StateImageIndex == CheckedState }); + } + } + if (levels.Count > 0) hierarchy["levels"] = levels; + hierarchies.Add(hierarchy); + } + } + + if (columns.Count > 0) table["columns"] = columns; + if (measures.Count > 0) table["measures"] = measures; + if (hierarchies.Count > 0) table["hierarchies"] = hierarchies; + tables.Add(table); + } + + return new JObject { ["tables"] = tables }; + } + + private static string StatusText(TreeView tree, string startupWarning) + { + var total = 0; + var included = 0; + CountNodes(tree.Nodes, ref total, ref included); + return included + " included / " + total + " objects" + + (string.IsNullOrWhiteSpace(startupWarning) ? "" : " " + startupWarning); + } + + private static void CountNodes(TreeNodeCollection nodes, ref int total, ref int included) + { + foreach (TreeNode node in nodes) + { + if (node.Tag is SchemaNode) + { + total++; + if (node.StateImageIndex != UncheckedState) included++; + } + CountNodes(node.Nodes, ref total, ref included); + } + } + + private static ImageList BuildStateImages() + { + var list = new ImageList { ImageSize = new Size(16, 16), ColorDepth = ColorDepth.Depth32Bit }; + list.Images.Add(DrawStateImage(UncheckedState)); + list.Images.Add(DrawStateImage(CheckedState)); + list.Images.Add(DrawStateImage(MixedState)); + return list; + } + + private static Bitmap DrawStateImage(int state) + { + var bmp = new Bitmap(16, 16); + using (var g = Graphics.FromImage(bmp)) + using (var border = new Pen(Color.FromArgb(120, 120, 120))) + using (var fill = new SolidBrush(Color.White)) + using (var mark = new Pen(Color.FromArgb(0, 120, 215), 2F)) + using (var mixed = new SolidBrush(Color.FromArgb(0, 120, 215))) + { + g.Clear(Color.Transparent); + g.FillRectangle(fill, 2, 2, 12, 12); + g.DrawRectangle(border, 2, 2, 12, 12); + if (state == CheckedState) + { + g.DrawLines(mark, new[] { new Point(4, 8), new Point(7, 11), new Point(12, 5) }); + } + else if (state == MixedState) + { + g.FillRectangle(mixed, 5, 7, 6, 2); + } + } + return bmp; + } + + private sealed class SchemaNode + { + public readonly string Kind; + public readonly string Table; + public readonly string Property; + public readonly string Hierarchy; + public readonly string Level; + + public SchemaNode(string kind, string table, string property, string hierarchy, string level) + { + Kind = kind; + Table = table; + Property = property; + Hierarchy = hierarchy; + Level = level; + } + } + + private static Culture EnsureCulture(TabularEditor.TOMWrapper.Model model, string cultureName, out string warning) + { + warning = null; + + if (model.Cultures.Contains(cultureName)) return model.Cultures[cultureName]; + + try + { + return model.AddTranslation(cultureName); + } + catch + { + // Power BI Desktop-connected models may block AddTranslation. Fall back to TE's import helper. + } + + try + { + if (TryImportEmptyCulture(model, cultureName) && model.Cultures.Contains(cultureName)) + { + warning = "Created " + cultureName + " culture."; + return model.Cultures[cultureName]; + } + } + catch + { + // Fall through to an existing culture or a controlled message. + } + + var fallback = model.Cultures.FirstOrDefault(c => !string.IsNullOrWhiteSpace(c.Content)) + ?? model.Cultures.FirstOrDefault(); + if (fallback != null) + { + warning = "Could not create " + cultureName + "; using " + fallback.Name + "."; + return fallback; + } + + return null; + } + + private static bool TryImportEmptyCulture(TabularEditor.TOMWrapper.Model model, string cultureName) + { + var helperType = typeof(TabularEditor.TOMWrapper.Model).Assembly.GetType("TabularEditor.TOMWrapper.TabularCultureHelper"); + if (helperType == null) return false; + + var method = helperType.GetMethod("ImportCulture", BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); + if (method == null) return false; + + var cultureJson = new JObject { ["name"] = cultureName }; + var result = method.Invoke(null, new object[] { cultureJson, model, false, true }); + return result is bool ok && ok; + } + + private static JObject GetSchema(Culture culture) + { + var payload = GetPayload(culture, false); + return SchemaFromEntities(payload["Entities"] as JObject); + } + + private static void SetSchema(Culture culture, JObject schema) + { + var payload = GetPayload(culture, true); + payload["Entities"] = EntitiesFromSchema(schema); + SavePayload(culture, payload); + } + + private static JObject GetPayload(Culture culture, bool create) + { + if (!string.IsNullOrWhiteSpace(culture.Content)) + { + return JObject.Parse(culture.Content); + } + + if (!create) return new JObject(); + + return new JObject + { + ["Version"] = "4.2.0", + ["Language"] = culture.Name, + ["Entities"] = new JObject(), + ["Agents"] = new JObject + { + ["Internal"] = new JObject { ["Version"] = "1.1.0" } + } + }; + } + + private static void SavePayload(Culture culture, JObject payload) + { + culture.Content = payload.ToString(Formatting.Indented); + } + + private static JObject SchemaFromEntities(JObject entities) + { + var tableMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + var orderedTables = new JArray(); + + if (entities == null) return new JObject { ["tables"] = orderedTables }; + + foreach (var property in entities.Properties()) + { + var entity = property.Value as JObject; + if (entity == null) continue; + + var binding = BindingFromEntity(entity); + if (binding == null) continue; + + var tableName = StringValue(binding, "ConceptualEntity"); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + var include = EntityIncluded(entity); + var table = GetOrAddTable(tableMap, orderedTables, tableName); + var propertyName = StringValue(binding, "ConceptualProperty"); + var hierarchyName = StringValue(binding, "Hierarchy"); + var levelName = StringValue(binding, "HierarchyLevel"); + + if (!string.IsNullOrWhiteSpace(levelName)) + { + var hierarchy = GetOrAddHierarchy(table, hierarchyName); + GetArray(hierarchy, "levels").Add(new JObject { ["name"] = levelName, ["include"] = include }); + } + else if (!string.IsNullOrWhiteSpace(hierarchyName)) + { + var hierarchy = GetOrAddHierarchy(table, hierarchyName); + hierarchy["include"] = include; + } + else if (!string.IsNullOrWhiteSpace(propertyName)) + { + GetArray(table, "columns").Add(new JObject { ["name"] = propertyName, ["include"] = include }); + } + else + { + table["include"] = include; + } + } + + RemoveEmptyArrays(orderedTables); + return new JObject { ["tables"] = orderedTables }; + } + + private static JObject EntitiesFromSchema(JObject schema) + { + var entities = new JObject(); + var usedIds = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var tableEntry in CollectionEntries(schema["tables"] ?? schema["Tables"])) + { + var table = tableEntry.Value as JObject; + var tableName = SchemaObjectName(tableEntry.Key, tableEntry.Value); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + AddEntity(entities, usedIds, tableName, IncludeValue(tableEntry.Value), tableName, null, null, null); + + foreach (var columnEntry in CollectionEntries(table?["columns"] ?? table?["Columns"])) + { + var columnName = SchemaObjectName(columnEntry.Key, columnEntry.Value); + if (!string.IsNullOrWhiteSpace(columnName)) AddEntity(entities, usedIds, tableName + "_" + columnName, IncludeValue(columnEntry.Value), tableName, columnName, null, null); + } + + foreach (var measureEntry in CollectionEntries(table?["measures"] ?? table?["Measures"])) + { + var measureName = SchemaObjectName(measureEntry.Key, measureEntry.Value); + if (!string.IsNullOrWhiteSpace(measureName)) AddEntity(entities, usedIds, tableName + "_" + measureName, IncludeValue(measureEntry.Value), tableName, measureName, null, null); + } + + foreach (var hierarchyEntry in CollectionEntries(table?["hierarchies"] ?? table?["Hierarchies"])) + { + var hierarchy = hierarchyEntry.Value as JObject; + var hierarchyName = SchemaObjectName(hierarchyEntry.Key, hierarchyEntry.Value); + if (string.IsNullOrWhiteSpace(hierarchyName)) continue; + + AddEntity(entities, usedIds, tableName + "_" + hierarchyName, IncludeValue(hierarchyEntry.Value), tableName, null, hierarchyName, null); + + foreach (var levelEntry in CollectionEntries(hierarchy?["levels"] ?? hierarchy?["Levels"])) + { + var levelName = SchemaObjectName(levelEntry.Key, levelEntry.Value); + if (!string.IsNullOrWhiteSpace(levelName)) AddEntity(entities, usedIds, tableName + "_" + hierarchyName + "_" + levelName, IncludeValue(levelEntry.Value), tableName, null, hierarchyName, levelName); + } + } + } + + return entities; + } + + private static JObject BindingFromEntity(JObject entity) + { + if (entity["Binding"] is JObject binding) return binding; + if (entity["Definition"] is JObject definition && definition["Binding"] is JObject nestedBinding) return nestedBinding; + return null; + } + + private static bool EntityIncluded(JObject entity) + { + var state = StringValue(entity, "State") ?? "Generated"; + var normalized = state.Trim().ToLowerInvariant(); + return normalized != "deleted" && normalized != "hidden" && normalized != "disabled"; + } + + private static string StringValue(JObject obj, string name) + { + return (string)(obj[name] ?? obj[Char.ToLowerInvariant(name[0]) + name.Substring(1)]); + } + + private static JObject GetOrAddTable(Dictionary tableMap, JArray orderedTables, string tableName) + { + if (tableMap.TryGetValue(tableName, out var table)) return table; + + table = new JObject + { + ["name"] = tableName, + ["include"] = true, + ["columns"] = new JArray(), + ["hierarchies"] = new JArray() + }; + tableMap[tableName] = table; + orderedTables.Add(table); + return table; + } + + private static JObject GetOrAddHierarchy(JObject table, string hierarchyName) + { + var name = hierarchyName ?? ""; + var hierarchies = GetArray(table, "hierarchies"); + foreach (var existing in hierarchies.OfType()) + { + if (string.Equals((string)existing["name"], name, StringComparison.OrdinalIgnoreCase)) return existing; + } + + var hierarchy = new JObject + { + ["name"] = name, + ["include"] = true, + ["levels"] = new JArray() + }; + hierarchies.Add(hierarchy); + return hierarchy; + } + + private static JArray GetArray(JObject obj, string propertyName) + { + if (!(obj[propertyName] is JArray array)) + { + array = new JArray(); + obj[propertyName] = array; + } + return array; + } + + private static void RemoveEmptyArrays(JArray tables) + { + foreach (var table in tables.OfType()) + { + if (table["columns"] is JArray columns && columns.Count == 0) table.Remove("columns"); + if (table["hierarchies"] is JArray hierarchies) + { + foreach (var hierarchy in hierarchies.OfType()) + { + if (hierarchy["levels"] is JArray levels && levels.Count == 0) hierarchy.Remove("levels"); + } + if (hierarchies.Count == 0) table.Remove("hierarchies"); + } + } + } + + private static IEnumerable> CollectionEntries(JToken value) + { + if (value is JArray array) + { + foreach (var item in array) + { + yield return new KeyValuePair(SchemaObjectName(null, item), item); + } + yield break; + } + + if (value is JObject obj) + { + foreach (var property in obj.Properties()) + { + yield return new KeyValuePair(property.Name, property.Value); + } + } + } + + private static string SchemaObjectName(string key, JToken value) + { + if (value is JObject obj) + { + return (string)(obj["name"] ?? obj["Name"] ?? obj["id"] ?? obj["Id"]) ?? key; + } + return key; + } + + private static bool IncludeValue(JToken value) + { + if (value != null && value.Type == JTokenType.Boolean) return (bool)value; + + if (value is JObject obj) + { + var include = obj["include"] ?? obj["Include"] ?? obj["enabled"] ?? obj["Enabled"] ?? obj["selected"] ?? obj["Selected"]; + if (include != null && include.Type == JTokenType.Boolean) return (bool)include; + + var visibility = ((string)(obj["visibility"] ?? obj["Visibility"]) ?? "").Trim().ToLowerInvariant(); + if (visibility == "hidden") return false; + if (visibility == "visible") return true; + } + + return true; + } + + private static void AddEntity(JObject entities, HashSet usedIds, string rawId, bool include, string table, string property, string hierarchy, string level) + { + var binding = new JObject { ["ConceptualEntity"] = table }; + if (!string.IsNullOrWhiteSpace(property)) binding["ConceptualProperty"] = property; + if (!string.IsNullOrWhiteSpace(hierarchy)) binding["Hierarchy"] = hierarchy; + if (!string.IsNullOrWhiteSpace(level)) binding["HierarchyLevel"] = level; + + entities[UniqueEntityId(rawId, usedIds)] = new JObject + { + ["Binding"] = binding, + ["State"] = include ? "Generated" : "Hidden" + }; + } + + private static string UniqueEntityId(string raw, HashSet usedIds) + { + var baseId = Regex.Replace((raw ?? "entity").Trim().ToLowerInvariant(), "[^a-z0-9]+", "_").Trim('_'); + if (string.IsNullOrWhiteSpace(baseId)) baseId = "entity"; + + var candidate = baseId; + var index = 2; + while (usedIds.Contains(candidate)) + { + candidate = baseId + "_" + index; + index++; + } + usedIds.Add(candidate); + return candidate; + } + + private static string NormalizeForEditor(string text) + { + return (text ?? "").Replace("\r\n", "\n").Replace("\r", "\n").Replace("\n", Environment.NewLine); + } + + public static string RootMessage(Exception ex) + { + if (ex == null) return ""; + while (ex.InnerException != null) ex = ex.InnerException; + return ex.Message; + } +} + +public sealed class ScriptTextEditor : IDisposable +{ + private readonly TextBox textBox; + + public Control Control { get; private set; } + public event EventHandler TextChanged; + + private ScriptTextEditor(TextBox textBox) + { + this.textBox = textBox; + Control = textBox; + Control.TextChanged += (sender, args) => TextChanged?.Invoke(this, EventArgs.Empty); + } + + public static ScriptTextEditor Create(bool wordWrap) + { + var textBox = new TextBox + { + Dock = DockStyle.Fill, + Multiline = true, + ScrollBars = ScrollBars.Both, + WordWrap = wordWrap, + AcceptsReturn = true, + AcceptsTab = true, + Font = new Font("Consolas", 10F), + BorderStyle = BorderStyle.None + }; + return new ScriptTextEditor(textBox); + } + + public string Text + { + get { return Control.Text ?? ""; } + set { Control.Text = value ?? ""; } + } + + public bool WordWrap + { + get { return textBox.WordWrap; } + set + { + textBox.WordWrap = value; + } + } + + public void SelectStart() + { + textBox.SelectionStart = 0; + textBox.SelectionLength = 0; + } + + public void Dispose() + { + Control?.Dispose(); + } +} diff --git a/Semantic Model AI Metadata/Manage Semantic Model AI Metadata.csx b/Semantic Model AI Metadata/Manage Semantic Model AI Metadata.csx new file mode 100644 index 0000000..ddbad5f --- /dev/null +++ b/Semantic Model AI Metadata/Manage Semantic Model AI Metadata.csx @@ -0,0 +1,628 @@ +/* + * Title: Manage semantic model AI metadata + * + * Author: Kurt Buhler + * + * This script reads, sets, lists, or deletes semantic model AI instructions + * and AI schema through culture linguistic metadata. AI instructions map to + * CustomInstructions, and AI schema maps to Entities. + * + * It is designed for non-interactive `te script` automation and accepts its + * action, target, culture, and payload through environment variables. + * + * Provided as-is. This script edits culture linguistic metadata directly and is + * not an official supported Tabular Editor product feature. + */ + +// Manage semantic model AI instructions and AI schema from te script. +// +// Non-interactive usage: +// TE_AI_ACTION=get TE_AI_TARGET=both te script -S manage-ai-metadata.csx -m ./model --output-format json +// TE_AI_ACTION=set TE_AI_TARGET=instructions TE_AI_INPUT_FILE=./instructions.md te script -S manage-ai-metadata.csx -m ./model --save +// TE_AI_ACTION=set TE_AI_TARGET=schema TE_AI_INPUT_FILE=./schema.json te script -S manage-ai-metadata.csx -m ./model --save +// TE_AI_ACTION=delete TE_AI_TARGET=schema te script -S manage-ai-metadata.csx -m ./model --save +// +// Environment variables: +// TE_AI_ACTION list | get | set | delete. Default: get. +// TE_AI_TARGET instructions | schema | both. Default: both for get/list, required for set/delete. +// TE_AI_CULTURE Culture name to use. Default: first culture with linguistic metadata, then first culture, then en-US on set. +// TE_AI_INPUT_FILE File to read for set. +// TE_AI_INPUT Inline payload to use for set when TE_AI_INPUT_FILE is not set. +// TE_AI_OUTPUT_FILE Optional file path for JSON/text output. +// TE_AI_ALLOW_OVER_LIMIT=true permits instructions longer than 10000 characters. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using TabularEditor.TOMWrapper; + +var action = AiMetadata.Env("TE_AI_ACTION", "get").Trim().ToLowerInvariant(); +var target = AiMetadata.Env("TE_AI_TARGET", action == "set" || action == "delete" ? "" : "both").Trim().ToLowerInvariant(); +var cultureName = AiMetadata.Env("TE_AI_CULTURE", "").Trim(); +var outputFile = AiMetadata.Env("TE_AI_OUTPUT_FILE", "").Trim(); + +// Write a machine-readable error envelope (stdout / TE_AI_OUTPUT_FILE) before +// reporting the error, so failures never leave a stale success payload behind. +// The envelope write must never mask the original error with its own exception +// (e.g. an unwritable TE_AI_OUTPUT_FILE), so it is best-effort. +void Fail(string message) +{ + try + { + AiMetadata.WriteResult(new JObject + { + ["error"] = message, + ["action"] = action, + ["target"] = target, + ["culture"] = cultureName + }, outputFile); + } + catch (Exception writeEx) + { + Error("Failed to write error envelope: " + writeEx.Message); + } + Error(message); +} + +if (action != "list" && action != "get" && action != "set" && action != "delete") +{ + Fail("TE_AI_ACTION must be 'list', 'get', 'set', or 'delete'."); + return; +} + +try +{ + if (action == "list") + { + AiMetadata.WriteResult(AiMetadata.ListCultures(Model), outputFile); + return; + } + + if (target != "instructions" && target != "schema" && target != "both") + { + Fail("TE_AI_TARGET must be 'instructions', 'schema', or 'both'."); + return; + } + + var culture = AiMetadata.FindCulture(Model, cultureName, action == "set"); + if (culture == null) + { + Fail("No culture is available on this model. Add a culture before managing AI metadata."); + return; + } + + if (action == "get") + { + var result = AiMetadata.Read(Model, culture, target); + AiMetadata.WriteResult(result, outputFile); + return; + } + + if (action == "set") + { + var input = AiMetadata.ReadInput(); + if (target == "instructions") + { + if (input.Length > AiMetadata.InstructionsLimit && !AiMetadata.AllowOverLimit()) + { + Fail("AI instructions are " + input.Length + " characters. Limit is " + AiMetadata.InstructionsLimit + ". Set TE_AI_ALLOW_OVER_LIMIT=true to override."); + return; + } + AiMetadata.SetInstructions(culture, input); + } + else if (target == "schema") + { + var schema = AiMetadata.ResolveSchemaInput(JObject.Parse(input)); + if (schema == null) + { + Fail("No tables found in input. Expected {\"tables\": [...]} or the get output envelope."); + return; + } + AiMetadata.SetSchema(culture, schema); + } + else + { + Fail("TE_AI_TARGET=both is not valid for set. Set instructions and schema in separate calls."); + return; + } + + AiMetadata.WriteResult(AiMetadata.Read(Model, culture, target), outputFile); + return; + } + + if (action == "delete") + { + if (target == "instructions" || target == "both") AiMetadata.DeleteInstructions(culture); + if (target == "schema" || target == "both") AiMetadata.DeleteSchema(culture); + AiMetadata.WriteResult(AiMetadata.Read(Model, culture, target), outputFile); + return; + } +} +catch (Exception ex) +{ + Fail(ex.Message); +} + +public static class AiMetadata +{ + public const int InstructionsLimit = 10000; + + public static string Env(string name, string fallback) + { + var value = Environment.GetEnvironmentVariable(name); + return string.IsNullOrWhiteSpace(value) ? fallback : value; + } + + public static bool AllowOverLimit() + { + return string.Equals(Env("TE_AI_ALLOW_OVER_LIMIT", ""), "true", StringComparison.OrdinalIgnoreCase); + } + + public static string ReadInput() + { + var inputFile = Env("TE_AI_INPUT_FILE", "").Trim(); + if (!string.IsNullOrWhiteSpace(inputFile)) return File.ReadAllText(inputFile); + + var input = Environment.GetEnvironmentVariable("TE_AI_INPUT"); + if (input != null) return input; + + throw new InvalidOperationException("Set TE_AI_INPUT_FILE or TE_AI_INPUT for TE_AI_ACTION=set."); + } + + public static void WriteResult(JToken result, string outputFile) + { + var text = result.ToString(Formatting.Indented); + if (!string.IsNullOrWhiteSpace(outputFile)) + { + var dir = Path.GetDirectoryName(Path.GetFullPath(outputFile)); + if (!string.IsNullOrEmpty(dir)) Directory.CreateDirectory(dir); + File.WriteAllText(outputFile, text + Environment.NewLine); + Console.WriteLine("Wrote " + outputFile); + return; + } + + Console.WriteLine(text); + } + + public static Culture FindCulture(TabularEditor.TOMWrapper.Model model, string cultureName, bool createIfMissing) + { + if (!string.IsNullOrWhiteSpace(cultureName)) + { + if (!model.Cultures.Contains(cultureName)) + { + if (createIfMissing) return model.AddTranslation(cultureName); + throw new InvalidOperationException("Culture '" + cultureName + "' was not found."); + } + return model.Cultures[cultureName]; + } + + var withMetadata = model.Cultures.FirstOrDefault(c => !string.IsNullOrWhiteSpace(c.Content)); + if (withMetadata != null) return withMetadata; + var firstCulture = model.Cultures.FirstOrDefault(); + if (firstCulture != null) return firstCulture; + return createIfMissing ? model.AddTranslation("en-US") : null; + } + + public static JArray ListCultures(TabularEditor.TOMWrapper.Model model) + { + var cultures = new JArray(); + foreach (var culture in model.Cultures) + { + var payload = TryParsePayload(culture); + var entities = payload?["Entities"] as JObject; + cultures.Add(new JObject + { + ["name"] = culture.Name, + ["hasLinguisticMetadata"] = !string.IsNullOrWhiteSpace(culture.Content), + ["hasAiInstructions"] = payload?["CustomInstructions"] != null, + ["schemaObjectCount"] = entities == null ? 0 : entities.Properties().Count() + }); + } + return cultures; + } + + public static JObject Read(TabularEditor.TOMWrapper.Model model, Culture culture, string target) + { + var payload = GetPayload(culture, false); + var result = new JObject + { + ["model"] = model.Name, + ["culture"] = culture.Name, + ["storage"] = "culture.linguisticMetadata", + ["copilotTooling"] = HasCopilotTooling(model) + }; + + if (target == "instructions" || target == "both") + { + var instructions = (string)payload["CustomInstructions"]; + result["aiInstructions"] = new JObject + { + ["exists"] = instructions != null, + ["length"] = instructions == null ? 0 : instructions.Length, + ["limit"] = InstructionsLimit, + ["text"] = instructions ?? "" + }; + } + + if (target == "schema" || target == "both") + { + var schema = SchemaFromEntities(payload["Entities"] as JObject); + result["aiSchema"] = schema; + result["schemaObjectCount"] = CountSchemaObjects(schema); + } + + return result; + } + + public static void SetInstructions(Culture culture, string instructions) + { + var payload = GetPayload(culture, true); + payload["CustomInstructions"] = instructions ?? ""; + SavePayload(culture, payload); + } + + public static void DeleteInstructions(Culture culture) + { + if (string.IsNullOrWhiteSpace(culture.Content)) return; + var payload = GetPayload(culture, false); + if (!payload.Remove("CustomInstructions")) return; + SavePayload(culture, payload); + } + + public static JObject ResolveSchemaInput(JObject input) + { + if (input == null) return null; + if (HasTablesCollection(input)) return input; + if (input["aiSchema"] is JObject envelope && HasTablesCollection(envelope)) return envelope; + return null; + } + + private static bool HasTablesCollection(JObject candidate) + { + // A JSON null value parses to a JValue, not a reference null, so a + // bare null check would let {"tables": null} through and wipe Entities. + return candidate["tables"] is JContainer || candidate["Tables"] is JContainer; + } + + public static void SetSchema(Culture culture, JObject schema) + { + var payload = GetPayload(culture, true); + payload["Entities"] = EntitiesFromSchema(schema); + SavePayload(culture, payload); + } + + public static void DeleteSchema(Culture culture) + { + if (string.IsNullOrWhiteSpace(culture.Content)) return; + var payload = GetPayload(culture, false); + if (!payload.Remove("Entities")) return; + SavePayload(culture, payload); + } + + private static bool HasCopilotTooling(TabularEditor.TOMWrapper.Model model) + { + var value = model.GetAnnotation("PBI_ProTooling"); + return value != null && value.IndexOf("CopilotTooling", StringComparison.OrdinalIgnoreCase) >= 0; + } + + private static JObject TryParsePayload(Culture culture) + { + if (string.IsNullOrWhiteSpace(culture.Content)) return null; + try + { + return JObject.Parse(culture.Content); + } + catch + { + return null; + } + } + + private static JObject GetPayload(Culture culture, bool create) + { + if (!string.IsNullOrWhiteSpace(culture.Content)) + { + try + { + return JObject.Parse(culture.Content); + } + catch (Exception ex) + { + throw new InvalidOperationException("Culture '" + culture.Name + "' linguistic metadata is not valid JSON: " + ex.Message); + } + } + + if (!create) + { + return new JObject(); + } + + return new JObject + { + ["Version"] = "4.2.0", + ["Language"] = culture.Name, + ["Entities"] = new JObject(), + ["Agents"] = new JObject + { + ["Internal"] = new JObject { ["Version"] = "1.1.0" } + } + }; + } + + private static void SavePayload(Culture culture, JObject payload) + { + culture.Content = payload.ToString(Formatting.Indented); + } + + private static JObject SchemaFromEntities(JObject entities) + { + var tableMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + var orderedTables = new JArray(); + + if (entities == null) + { + return new JObject { ["tables"] = orderedTables }; + } + + foreach (var property in entities.Properties()) + { + var entity = property.Value as JObject; + if (entity == null) continue; + + var binding = BindingFromEntity(entity); + if (binding == null) continue; + + var tableName = StringValue(binding, "ConceptualEntity"); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + var include = EntityIncluded(entity); + var table = GetOrAddTable(tableMap, orderedTables, tableName); + var propertyName = StringValue(binding, "ConceptualProperty"); + var hierarchyName = StringValue(binding, "Hierarchy"); + var levelName = StringValue(binding, "HierarchyLevel"); + + if (!string.IsNullOrWhiteSpace(levelName)) + { + var hierarchy = GetOrAddHierarchy(table, hierarchyName); + GetArray(hierarchy, "levels").Add(new JObject { ["name"] = levelName, ["include"] = include }); + } + else if (!string.IsNullOrWhiteSpace(hierarchyName)) + { + var hierarchy = GetOrAddHierarchy(table, hierarchyName); + hierarchy["include"] = include; + } + else if (!string.IsNullOrWhiteSpace(propertyName)) + { + GetArray(table, "columns").Add(new JObject { ["name"] = propertyName, ["include"] = include }); + } + else + { + table["include"] = include; + } + } + + RemoveEmptyArrays(orderedTables); + return new JObject { ["tables"] = orderedTables }; + } + + private static JObject EntitiesFromSchema(JObject schema) + { + var entities = new JObject(); + var usedIds = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var tableEntry in CollectionEntries(schema["tables"] ?? schema["Tables"])) + { + var table = tableEntry.Value as JObject; + var tableName = SchemaObjectName(tableEntry.Key, tableEntry.Value); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + AddEntity(entities, usedIds, tableName, IncludeValue(tableEntry.Value), tableName, null, null, null); + + foreach (var columnEntry in CollectionEntries(table?["columns"] ?? table?["Columns"])) + { + var columnName = SchemaObjectName(columnEntry.Key, columnEntry.Value); + if (!string.IsNullOrWhiteSpace(columnName)) AddEntity(entities, usedIds, tableName + "_" + columnName, IncludeValue(columnEntry.Value), tableName, columnName, null, null); + } + + foreach (var measureEntry in CollectionEntries(table?["measures"] ?? table?["Measures"])) + { + var measureName = SchemaObjectName(measureEntry.Key, measureEntry.Value); + if (!string.IsNullOrWhiteSpace(measureName)) AddEntity(entities, usedIds, tableName + "_" + measureName, IncludeValue(measureEntry.Value), tableName, measureName, null, null); + } + + foreach (var hierarchyEntry in CollectionEntries(table?["hierarchies"] ?? table?["Hierarchies"])) + { + var hierarchy = hierarchyEntry.Value as JObject; + var hierarchyName = SchemaObjectName(hierarchyEntry.Key, hierarchyEntry.Value); + if (string.IsNullOrWhiteSpace(hierarchyName)) continue; + + AddEntity(entities, usedIds, tableName + "_" + hierarchyName, IncludeValue(hierarchyEntry.Value), tableName, null, hierarchyName, null); + + foreach (var levelEntry in CollectionEntries(hierarchy?["levels"] ?? hierarchy?["Levels"])) + { + var levelName = SchemaObjectName(levelEntry.Key, levelEntry.Value); + if (!string.IsNullOrWhiteSpace(levelName)) AddEntity(entities, usedIds, tableName + "_" + hierarchyName + "_" + levelName, IncludeValue(levelEntry.Value), tableName, null, hierarchyName, levelName); + } + } + } + + return entities; + } + + private static JObject BindingFromEntity(JObject entity) + { + if (entity["Binding"] is JObject binding) return binding; + if (entity["Definition"] is JObject definition && definition["Binding"] is JObject nestedBinding) return nestedBinding; + return null; + } + + private static bool EntityIncluded(JObject entity) + { + var state = StringValue(entity, "State") ?? "Generated"; + var normalized = state.Trim().ToLowerInvariant(); + return normalized != "deleted" && normalized != "hidden" && normalized != "disabled"; + } + + private static string StringValue(JObject obj, string name) + { + return (string)(obj[name] ?? obj[Char.ToLowerInvariant(name[0]) + name.Substring(1)]); + } + + private static JObject GetOrAddTable(Dictionary tableMap, JArray orderedTables, string tableName) + { + if (tableMap.TryGetValue(tableName, out var table)) return table; + + table = new JObject + { + ["name"] = tableName, + ["include"] = true, + ["columns"] = new JArray(), + ["hierarchies"] = new JArray() + }; + tableMap[tableName] = table; + orderedTables.Add(table); + return table; + } + + private static JObject GetOrAddHierarchy(JObject table, string hierarchyName) + { + var name = hierarchyName ?? ""; + var hierarchies = GetArray(table, "hierarchies"); + foreach (var existing in hierarchies.OfType()) + { + if (string.Equals((string)existing["name"], name, StringComparison.OrdinalIgnoreCase)) return existing; + } + + var hierarchy = new JObject + { + ["name"] = name, + ["include"] = true, + ["levels"] = new JArray() + }; + hierarchies.Add(hierarchy); + return hierarchy; + } + + private static JArray GetArray(JObject obj, string propertyName) + { + if (!(obj[propertyName] is JArray array)) + { + array = new JArray(); + obj[propertyName] = array; + } + return array; + } + + private static void RemoveEmptyArrays(JArray tables) + { + foreach (var table in tables.OfType()) + { + if (table["columns"] is JArray columns && columns.Count == 0) table.Remove("columns"); + if (table["hierarchies"] is JArray hierarchies) + { + foreach (var hierarchy in hierarchies.OfType()) + { + if (hierarchy["levels"] is JArray levels && levels.Count == 0) hierarchy.Remove("levels"); + } + if (hierarchies.Count == 0) table.Remove("hierarchies"); + } + } + } + + private static IEnumerable> CollectionEntries(JToken value) + { + if (value is JArray array) + { + foreach (var item in array) + { + yield return new KeyValuePair(SchemaObjectName(null, item), item); + } + yield break; + } + + if (value is JObject obj) + { + foreach (var property in obj.Properties()) + { + yield return new KeyValuePair(property.Name, property.Value); + } + } + } + + private static string SchemaObjectName(string key, JToken value) + { + if (value is JObject obj) + { + return (string)(obj["name"] ?? obj["Name"] ?? obj["id"] ?? obj["Id"]) ?? key; + } + return key; + } + + private static bool IncludeValue(JToken value) + { + if (value != null && value.Type == JTokenType.Boolean) return (bool)value; + + if (value is JObject obj) + { + var include = obj["include"] ?? obj["Include"] ?? obj["enabled"] ?? obj["Enabled"] ?? obj["selected"] ?? obj["Selected"]; + if (include != null && include.Type == JTokenType.Boolean) return (bool)include; + + var visibility = ((string)(obj["visibility"] ?? obj["Visibility"]) ?? "").Trim().ToLowerInvariant(); + if (visibility == "hidden") return false; + if (visibility == "visible") return true; + } + + return true; + } + + private static void AddEntity(JObject entities, HashSet usedIds, string rawId, bool include, string table, string property, string hierarchy, string level) + { + var binding = new JObject { ["ConceptualEntity"] = table }; + if (!string.IsNullOrWhiteSpace(property)) binding["ConceptualProperty"] = property; + if (!string.IsNullOrWhiteSpace(hierarchy)) binding["Hierarchy"] = hierarchy; + if (!string.IsNullOrWhiteSpace(level)) binding["HierarchyLevel"] = level; + + entities[UniqueEntityId(rawId, usedIds)] = new JObject + { + ["Binding"] = binding, + ["State"] = include ? "Generated" : "Hidden" + }; + } + + private static string UniqueEntityId(string raw, HashSet usedIds) + { + var baseId = Regex.Replace((raw ?? "entity").Trim().ToLowerInvariant(), "[^a-z0-9]+", "_").Trim('_'); + if (string.IsNullOrWhiteSpace(baseId)) baseId = "entity"; + + var candidate = baseId; + var index = 2; + while (usedIds.Contains(candidate)) + { + candidate = baseId + "_" + index; + index++; + } + usedIds.Add(candidate); + return candidate; + } + + private static int CountSchemaObjects(JObject schema) + { + var count = 0; + foreach (var table in (schema["tables"] as JArray ?? new JArray()).OfType()) + { + count++; + count += (table["columns"] as JArray ?? new JArray()).Count; + count += (table["measures"] as JArray ?? new JArray()).Count; + foreach (var hierarchy in (table["hierarchies"] as JArray ?? new JArray()).OfType()) + { + count++; + count += (hierarchy["levels"] as JArray ?? new JArray()).Count; + } + } + return count; + } +} diff --git a/Semantic Model AI Metadata/README.md b/Semantic Model AI Metadata/README.md new file mode 100644 index 0000000..633b408 --- /dev/null +++ b/Semantic Model AI Metadata/README.md @@ -0,0 +1,81 @@ +# Semantic Model AI Metadata + +Scripts for managing semantic model AI instructions and AI schema stored in +Power BI / Analysis Services culture linguistic metadata. + +These scripts are provided as-is. They edit culture linguistic metadata directly +and are not official supported Tabular Editor product features. Test them on +copies or non-production models before using them on important models. + +The v1 VS Code extension exposes these as friendly files: + +- `Copilot/Instructions/instructions.md` +- `Copilot/schema.json` + +In the tabular model, those map to: + +- `CustomInstructions` for AI instructions +- `Entities` for the AI schema + +## Scripts + +- `Manage Semantic Model AI Metadata.csx`: non-interactive script for reading, + setting, listing, or deleting AI instructions and AI schema. Best used with + `te script`. +- `Edit Semantic Model AI Instructions.csx`: TE3 Desktop GUI editor for AI + instructions. It uses the connected model, defaults to `en-US`, and does not + require a selected object. It uses a standard WinForms multiline text box. +- `Edit Semantic Model AI Schema.csx`: TE3 Desktop GUI editor for AI schema. + It opens on an object tree similar to the perspective editor and includes a + JSON tab for exact roundtrips. The JSON tab uses a standard WinForms + multiline text box. + +## Non-interactive usage + +PowerShell: + +```powershell +$env:TE_AI_ACTION = "get" +$env:TE_AI_TARGET = "both" +te script -s "workspace" -d "model" ` + -S "Semantic Model AI Metadata/Manage Semantic Model AI Metadata.csx" ` + --output-format json --non-interactive +``` + +```powershell +$env:TE_AI_ACTION = "set" +$env:TE_AI_TARGET = "instructions" +$env:TE_AI_INPUT_FILE = "instructions.md" +te script -s "workspace" -d "model" ` + -S "Semantic Model AI Metadata/Manage Semantic Model AI Metadata.csx" ` + --save --output-format json --non-interactive +``` + +```powershell +$env:TE_AI_ACTION = "set" +$env:TE_AI_TARGET = "schema" +$env:TE_AI_INPUT_FILE = "schema.json" +te script -s "workspace" -d "model" ` + -S "Semantic Model AI Metadata/Manage Semantic Model AI Metadata.csx" ` + --save --output-format json --non-interactive +``` + +In bash/zsh, set the same variables inline before `te script`, for example: + +```bash +TE_AI_ACTION=get TE_AI_TARGET=both \ + te script -s "workspace" -d "model" \ + -S "Semantic Model AI Metadata/Manage Semantic Model AI Metadata.csx" \ + --output-format json --non-interactive +``` + +## Environment variables + +- `TE_AI_ACTION`: `list`, `get`, `set`, or `delete`. Default: `get`. +- `TE_AI_TARGET`: `instructions`, `schema`, or `both`. +- `TE_AI_CULTURE`: optional culture name. Defaults to the best available + culture and creates `en-US` on `set` when needed. +- `TE_AI_INPUT_FILE`: payload file for `set`. +- `TE_AI_INPUT`: inline payload for `set`. +- `TE_AI_OUTPUT_FILE`: optional output file. +- `TE_AI_ALLOW_OVER_LIMIT=true`: allow instructions over 10000 characters.