diff --git a/bindings/nodejs/Makefile b/bindings/nodejs/Makefile index f25ecc49..a0015c37 100644 --- a/bindings/nodejs/Makefile +++ b/bindings/nodejs/Makefile @@ -1,4 +1,22 @@ +REGISTRY := "http://localhost:4873" + wasm: - yarn exec napi build --target wasm32-wasip1-threads --output-dir npm/wasm32-wasi --release - mv npm/wasm32-wasi/browser.js . - mv npm/wasm32-wasi/zen-engine.wasm npm/wasm32-wasi/zen-engine.wasm32-wasi.wasm \ No newline at end of file + yarn build -- --target wasm32-wasip1-threads + mkdir -p npm/wasm32-wasi + cp zen-engine.wasm32-wasi.wasm zen-engine.wasi.cjs zen-engine.wasi-browser.js wasi-worker.mjs wasi-worker-browser.mjs npm/wasm32-wasi/ + +darwinArm: + yarn build -- --target aarch64-apple-darwin + mkdir -p npm/darwin-arm64 + cp zen-engine.darwin-arm64.node npm/darwin-arm64/ + +prepareLocalPublish: + yarn napi prepublish --skip-optional-publish + yarn createNpmDirs + +publish: + cd npm/wasm32-wasi && npm publish --registry $(REGISTRY) --ignore-scripts + cd npm/darwin-arm64 && npm publish --registry $(REGISTRY) --ignore-scripts + npm publish --registry $(REGISTRY) --ignore-scripts + +publishLocal: prepareLocalPublish wasm darwinArm publish diff --git a/bindings/nodejs/dts-header.d.ts b/bindings/nodejs/dts-header.d.ts index 04c990ba..293116bb 100644 --- a/bindings/nodejs/dts-header.d.ts +++ b/bindings/nodejs/dts-header.d.ts @@ -79,6 +79,141 @@ export type PolicyVariableType = | { type: 'object'; fields: Record } | { type: 'nullable'; inner: PolicyVariableType }; +/** Language-agnostic symbol key for an infix operator. The client maps the key to a localized phrase. */ +export type NlOpSym = + | 'gt' | 'gte' | 'lt' | 'lte' | 'eq' | 'ne' | 'in' | 'notIn' + | 'add' | 'sub' | 'mul' | 'div' | 'mod' | 'pow' + | 'and' | 'or' | 'not' | 'coalesce'; + +/** + * Structural separator word inside a multi-operand construct (conditional, closure, interval). + * `has` replaces `where` when an alias-elided closure body leads with a member on the binding + * ("any drivers has age less than 5"); clients should shorten the op label that follows it. + */ +export type NlWordSym = 'if' | 'then' | 'otherwise' | 'in' | 'where' | 'has' | 'rangeAnd'; + +/** + * Resolved type of a value/field token. `enum` / `array` carry an `index` + * into the per-result `enums` table; dedupe the domain once, reference by index. + */ +export type NlTypeTag = + | { t: 'number' } + | { t: 'string' } + | { t: 'bool' } + | { t: 'date' } + | { t: 'interval' } + | { t: 'object' } + | { t: 'null' } + | { t: 'unknown' } + | { t: 'enum'; index: number } + | { t: 'array'; items: NlTypeTag }; + +/** One operator the user may switch to: the symbol for labelling plus its ZEN source form for splicing. */ +export interface NlOpChoice { + sym: NlOpSym; + source: string; +} + +/** + * One enum domain value: `label` for display, `source` as the ready-to-splice ZEN literal. + * `source` is absent when the value contains both quote kinds and has no literal form. + */ +export interface NlEnumOption { + label: string; + source?: string; +} + +/** + * Widget hint present only when it adds information beyond the token's own type: enum domains + * (`options` indexes the `enums` table), expected dates, or the operator choices valid for the + * operand types — ordered comparisons only for number/date/unknown operands, eq/ne otherwise. + */ +export type NlEditHint = + | { kind: 'datePicker' } + | { kind: 'select'; options: number } + | { kind: 'multiSelect'; options: number } + | { kind: 'opSelect'; options: NlOpChoice[] }; + +/** + * One token in the symbolic NL stream. Structure is explicit via the + * group/list/interval markers and the infix placement of `op` / `word` tokens. + * `func.sym` / `method.sym` are stable camelCase keys (`'sum'`, `'format'`, `'d'` + * for the date constructor); `closure: true` marks a closure call whose operands + * follow as `element`, `word:'in'`, collection, `word:'where'`, body. + */ +export type NlTokenKind = + | { t: 'groupOpen' } + | { t: 'groupClose' } + | { t: 'listOpen' } + | { t: 'listClose' } + | { t: 'comma' } + | { t: 'enumList'; selected: string[] } + | { t: 'context' } + | { t: 'root' } + | { t: 'null' } + | { t: 'field'; path: string[]; ty: NlTypeTag } + | { t: 'element'; alias?: string } + | { t: 'number'; value: string } + | { t: 'str'; value: string } + | { t: 'bool'; value: boolean } + | { t: 'op'; sym: NlOpSym; implied: boolean; between: boolean } + | { t: 'word'; sym: NlWordSym } + | { t: 'assign' } + | { t: 'stmtEnd' } + | { t: 'func'; sym: string; closure: boolean } + | { t: 'method'; sym: string } + | { t: 'templateOpen' } + | { t: 'templateText'; value: string } + | { t: 'templateClose' } + | { t: 'intervalOpen'; inclusive: boolean } + | { t: 'intervalClose'; inclusive: boolean } + | { t: 'code'; source: string }; + +/** A projected token plus its source span (`[start, end)` bytes) and optional widget hint. */ +export interface NlToken { + token: NlTokenKind; + span: PolicySpan; + hint?: NlEditHint; +} + +export type NlDiagnosticSource = 'lexer' | 'parser' | 'typeCheck' | 'compiler'; + +export interface NlDiagnostic { + span: PolicySpan; + message: string; + severity: PolicySeverity; + source: NlDiagnosticSource; +} + +/** Result of projecting one expression; `enums` is the dedup table referenced by `NlTypeTag` / `NlEditHint` indices. */ +export interface NlResult { + id: string; + tokens: NlToken[]; + enums: NlEnumOption[][]; + diagnostics: NlDiagnostic[]; + /** Resolved `$` type for unary requests (decision-table input cells); present even for empty text. */ + subjectType?: PolicyVariableType; +} + +/** + * One projected expression/cell of a policy, returned by `PolicyWorkspace.nl(policyPath)`. + * The engine resolves scope + (for unary decision-table input cells) the subject type + * internally, so no `rootType` is supplied. `target` routes the result to the editor it + * came from (assertion condition, match arm, decision-table cell `{row, col}`, …). + */ +export interface PolicyNlExpression { + blockId: string; + target: PolicyCursorTarget; + kind: 'standard' | 'unary'; + /** Expression text this projection was computed from — compare against the editor value to detect staleness. */ + source: string; + tokens: NlToken[]; + enums: NlEnumOption[][]; + diagnostics: NlDiagnostic[]; + /** Resolved `$` type for unary cells. */ + subjectType?: PolicyVariableType; +} + /** * Wire-format block — the same shape passed into `setPolicy` / `updateBlock`. * The engine emits these inside `replaceBlock` / `insertBlock` edits so a diff --git a/bindings/nodejs/index.d.ts b/bindings/nodejs/index.d.ts index a37fd86e..8a415940 100644 --- a/bindings/nodejs/index.d.ts +++ b/bindings/nodejs/index.d.ts @@ -79,6 +79,141 @@ export type PolicyVariableType = | { type: 'object'; fields: Record } | { type: 'nullable'; inner: PolicyVariableType }; +/** Language-agnostic symbol key for an infix operator. The client maps the key to a localized phrase. */ +export type NlOpSym = + | 'gt' | 'gte' | 'lt' | 'lte' | 'eq' | 'ne' | 'in' | 'notIn' + | 'add' | 'sub' | 'mul' | 'div' | 'mod' | 'pow' + | 'and' | 'or' | 'not' | 'coalesce'; + +/** + * Structural separator word inside a multi-operand construct (conditional, closure, interval). + * `has` replaces `where` when an alias-elided closure body leads with a member on the binding + * ("any drivers has age less than 5"); clients should shorten the op label that follows it. + */ +export type NlWordSym = 'if' | 'then' | 'otherwise' | 'in' | 'where' | 'has' | 'rangeAnd'; + +/** + * Resolved type of a value/field token. `enum` / `array` carry an `index` + * into the per-result `enums` table; dedupe the domain once, reference by index. + */ +export type NlTypeTag = + | { t: 'number' } + | { t: 'string' } + | { t: 'bool' } + | { t: 'date' } + | { t: 'interval' } + | { t: 'object' } + | { t: 'null' } + | { t: 'unknown' } + | { t: 'enum'; index: number } + | { t: 'array'; items: NlTypeTag }; + +/** One operator the user may switch to: the symbol for labelling plus its ZEN source form for splicing. */ +export interface NlOpChoice { + sym: NlOpSym; + source: string; +} + +/** + * One enum domain value: `label` for display, `source` as the ready-to-splice ZEN literal. + * `source` is absent when the value contains both quote kinds and has no literal form. + */ +export interface NlEnumOption { + label: string; + source?: string; +} + +/** + * Widget hint present only when it adds information beyond the token's own type: enum domains + * (`options` indexes the `enums` table), expected dates, or the operator choices valid for the + * operand types — ordered comparisons only for number/date/unknown operands, eq/ne otherwise. + */ +export type NlEditHint = + | { kind: 'datePicker' } + | { kind: 'select'; options: number } + | { kind: 'multiSelect'; options: number } + | { kind: 'opSelect'; options: NlOpChoice[] }; + +/** + * One token in the symbolic NL stream. Structure is explicit via the + * group/list/interval markers and the infix placement of `op` / `word` tokens. + * `func.sym` / `method.sym` are stable camelCase keys (`'sum'`, `'format'`, `'d'` + * for the date constructor); `closure: true` marks a closure call whose operands + * follow as `element`, `word:'in'`, collection, `word:'where'`, body. + */ +export type NlTokenKind = + | { t: 'groupOpen' } + | { t: 'groupClose' } + | { t: 'listOpen' } + | { t: 'listClose' } + | { t: 'comma' } + | { t: 'enumList'; selected: string[] } + | { t: 'context' } + | { t: 'root' } + | { t: 'null' } + | { t: 'field'; path: string[]; ty: NlTypeTag } + | { t: 'element'; alias?: string } + | { t: 'number'; value: string } + | { t: 'str'; value: string } + | { t: 'bool'; value: boolean } + | { t: 'op'; sym: NlOpSym; implied: boolean; between: boolean } + | { t: 'word'; sym: NlWordSym } + | { t: 'assign' } + | { t: 'stmtEnd' } + | { t: 'func'; sym: string; closure: boolean } + | { t: 'method'; sym: string } + | { t: 'templateOpen' } + | { t: 'templateText'; value: string } + | { t: 'templateClose' } + | { t: 'intervalOpen'; inclusive: boolean } + | { t: 'intervalClose'; inclusive: boolean } + | { t: 'code'; source: string }; + +/** A projected token plus its source span (`[start, end)` bytes) and optional widget hint. */ +export interface NlToken { + token: NlTokenKind; + span: PolicySpan; + hint?: NlEditHint; +} + +export type NlDiagnosticSource = 'lexer' | 'parser' | 'typeCheck' | 'compiler'; + +export interface NlDiagnostic { + span: PolicySpan; + message: string; + severity: PolicySeverity; + source: NlDiagnosticSource; +} + +/** Result of projecting one expression; `enums` is the dedup table referenced by `NlTypeTag` / `NlEditHint` indices. */ +export interface NlResult { + id: string; + tokens: NlToken[]; + enums: NlEnumOption[][]; + diagnostics: NlDiagnostic[]; + /** Resolved `$` type for unary requests (decision-table input cells); present even for empty text. */ + subjectType?: PolicyVariableType; +} + +/** + * One projected expression/cell of a policy, returned by `PolicyWorkspace.nl(policyPath)`. + * The engine resolves scope + (for unary decision-table input cells) the subject type + * internally, so no `rootType` is supplied. `target` routes the result to the editor it + * came from (assertion condition, match arm, decision-table cell `{row, col}`, …). + */ +export interface PolicyNlExpression { + blockId: string; + target: PolicyCursorTarget; + kind: 'standard' | 'unary'; + /** Expression text this projection was computed from — compare against the editor value to detect staleness. */ + source: string; + tokens: NlToken[]; + enums: NlEnumOption[][]; + diagnostics: NlDiagnostic[]; + /** Resolved `$` type for unary cells. */ + subjectType?: PolicyVariableType; +} + /** * Wire-format block — the same shape passed into `setPolicy` / `updateBlock`. * The engine emits these inside `replaceBlock` / `insertBlock` edits so a @@ -255,6 +390,8 @@ export declare class PolicyWorkspace { outputs(req: PolicyScopeRequest): Array conditionalSchema(req: PolicyScopeRequest): PolicyConditionalSchema inspect(cursor: PolicyExpressionCursor): PolicyInspectResult | null + nl(policyPath: string): PolicyNlExpression[] + nlTokenize(cursor: PolicyExpressionCursor, text: string): NlResult | null completions(cursor: PolicyExpressionCursor): Array prepareRename(cursor: PolicyExpressionCursor): PolicyPrepareRenameResult | null /** @@ -349,6 +486,17 @@ export declare function evaluateUnaryExpression(expression: string, context: any export declare function evaluateUnaryExpressionSync(expression: string, context: any): boolean +export declare function nlEncodeString(value: string): string | null + +export declare function nlTokenizeBatch(requests: NlTokenizeRequest[], rootType: PolicyVariableType, strict?: boolean): NlResult[] + +export interface NlTokenizeRequest { + id: string + expression: string + unary: boolean + subjectType?: PolicyVariableType +} + export declare function overrideConfig(config: ZenConfig): void export interface PolicyCompletion { diff --git a/bindings/nodejs/src/expression.rs b/bindings/nodejs/src/expression.rs index 0fdc1d99..719ae93c 100644 --- a/bindings/nodejs/src/expression.rs +++ b/bindings/nodejs/src/expression.rs @@ -56,3 +56,118 @@ pub async fn render_template(template: String, context: Value) -> napi::Result Option { + zen_expression::nl::encode_string(&value) +} + +#[napi(object)] +pub struct NlTokenizeRequest { + pub id: String, + pub expression: String, + pub unary: bool, + #[napi(ts_type = "PolicyVariableType")] + pub subject_type: Option, +} + +#[napi( + ts_args_type = "requests: NlTokenizeRequest[], rootType: PolicyVariableType, strict?: boolean", + ts_return_type = "NlResult[]" +)] +pub fn nl_tokenize_batch( + requests: Vec, + root_type: Value, + strict: Option, +) -> napi::Result> { + use zen_expression::intellisense::IntelliSense; + use zen_expression::nl::NlRequest; + + let root = json_to_variable_type(&root_type); + let core_requests: Vec = requests + .into_iter() + .map(|request| NlRequest { + id: request.id, + expression: request.expression, + unary: request.unary, + subject_type: request.subject_type.as_ref().map(json_to_variable_type), + }) + .collect(); + + let mut intellisense = IntelliSense::new().with_strict(strict.unwrap_or(false)); + intellisense + .nl_tokenize_batch(&core_requests, &root) + .iter() + .map(|result| { + let mut value = serde_json::to_value(result) + .map_err(|e| napi::Error::from_reason(e.to_string()))?; + if let (Some(subject), Some(obj)) = (&result.subject_type, value.as_object_mut()) { + obj.insert( + "subjectType".into(), + crate::policy::variable_type_to_json(subject), + ); + } + Ok(value) + }) + .collect() +} + +fn json_to_variable_type(value: &Value) -> zen_expression::variable::VariableType { + use std::rc::Rc; + use zen_expression::variable::VariableType as VT; + + let Some(tag) = value.get("type").and_then(Value::as_str) else { + return VT::Any; + }; + + match tag { + "any" => VT::Any, + "null" => VT::Null, + "bool" => VT::Bool, + "string" => VT::String, + "number" => VT::Number, + "date" => VT::Date, + "interval" => VT::Interval, + "const" => value + .get("value") + .and_then(Value::as_str) + .map(|s| VT::Const(Rc::from(s))) + .unwrap_or(VT::Any), + "enum" => { + let name = value.get("name").and_then(Value::as_str).map(Rc::from); + let values = value + .get("values") + .and_then(Value::as_array) + .map(|arr| arr.iter().filter_map(Value::as_str).map(Rc::from).collect()) + .unwrap_or_default(); + VT::Enum(name, values) + } + "array" => { + let items = value + .get("items") + .map(json_to_variable_type) + .unwrap_or(VT::Any); + VT::Array(Rc::new(items)) + } + "object" => { + let object = VT::empty_object(); + if let (VT::Object(map), Some(fields)) = + (&object, value.get("fields").and_then(Value::as_object)) + { + for (key, field) in fields { + map.borrow_mut() + .insert(Rc::from(key.as_str()), json_to_variable_type(field)); + } + } + object + } + "nullable" => { + let inner = value + .get("inner") + .map(json_to_variable_type) + .unwrap_or(VT::Any); + VT::Nullable(Rc::new(inner)) + } + _ => VT::Any, + } +} diff --git a/bindings/nodejs/src/policy.rs b/bindings/nodejs/src/policy.rs index 47e0632c..ed68ea7d 100644 --- a/bindings/nodejs/src/policy.rs +++ b/bindings/nodejs/src/policy.rs @@ -331,7 +331,7 @@ fn resolve_diagnostic_cap(max: Option) -> usize { } } -fn variable_type_to_json(vt: &zen_expression::variable::VariableType) -> Value { +pub(crate) fn variable_type_to_json(vt: &zen_expression::variable::VariableType) -> Value { use zen_expression::variable::VariableType; match vt { @@ -591,6 +591,58 @@ impl PolicyWorkspace { })) } + #[napi(ts_return_type = "PolicyNlExpression[]")] + pub fn nl(&self, policy_path: String) -> napi::Result> { + self.inner + .nl(&policy_path) + .iter() + .map(|e| { + let mut value = serde_json::to_value(&e.result) + .map_err(|err| napi::Error::from_reason(err.to_string()))?; + let obj = value + .as_object_mut() + .ok_or_else(|| napi::Error::from_reason("nl result is not an object"))?; + obj.remove("id"); + obj.insert("blockId".into(), Value::String(e.block_id.to_string())); + obj.insert( + "kind".into(), + serde_json::to_value(e.kind) + .map_err(|err| napi::Error::from_reason(err.to_string()))?, + ); + obj.insert( + "target".into(), + serde_json::to_value(&e.target) + .map_err(|err| napi::Error::from_reason(err.to_string()))?, + ); + obj.insert("source".into(), Value::String(e.source.clone())); + if let Some(subject) = &e.result.subject_type { + obj.insert("subjectType".into(), variable_type_to_json(subject)); + } + Ok(value) + }) + .collect() + } + + #[napi(ts_return_type = "NlResult | null")] + pub fn nl_tokenize( + &self, + cursor: PolicyExpressionCursor, + text: String, + ) -> napi::Result> { + let cursor: policy::Cursor = cursor.try_into()?; + self.inner + .nl_tokenize(&cursor, &text) + .map(|result| { + let mut value = serde_json::to_value(&result) + .map_err(|err| napi::Error::from_reason(err.to_string()))?; + if let (Some(subject), Some(obj)) = (&result.subject_type, value.as_object_mut()) { + obj.insert("subjectType".into(), variable_type_to_json(subject)); + } + Ok(value) + }) + .transpose() + } + #[napi] pub fn completions( &self, diff --git a/bindings/nodejs/zen-engine.wasi-browser.js b/bindings/nodejs/zen-engine.wasi-browser.js index 22414dd4..bf2b47b5 100644 --- a/bindings/nodejs/zen-engine.wasi-browser.js +++ b/bindings/nodejs/zen-engine.wasi-browser.js @@ -65,6 +65,8 @@ export const evaluateExpression = __napiModule.exports.evaluateExpression export const evaluateExpressionSync = __napiModule.exports.evaluateExpressionSync export const evaluateUnaryExpression = __napiModule.exports.evaluateUnaryExpression export const evaluateUnaryExpressionSync = __napiModule.exports.evaluateUnaryExpressionSync +export const nlEncodeString = __napiModule.exports.nlEncodeString +export const nlTokenizeBatch = __napiModule.exports.nlTokenizeBatch export const overrideConfig = __napiModule.exports.overrideConfig export const renderTemplate = __napiModule.exports.renderTemplate export const renderTemplateSync = __napiModule.exports.renderTemplateSync diff --git a/bindings/nodejs/zen-engine.wasi.cjs b/bindings/nodejs/zen-engine.wasi.cjs index 939e88d4..956374e8 100644 --- a/bindings/nodejs/zen-engine.wasi.cjs +++ b/bindings/nodejs/zen-engine.wasi.cjs @@ -117,6 +117,8 @@ module.exports.evaluateExpression = __napiModule.exports.evaluateExpression module.exports.evaluateExpressionSync = __napiModule.exports.evaluateExpressionSync module.exports.evaluateUnaryExpression = __napiModule.exports.evaluateUnaryExpression module.exports.evaluateUnaryExpressionSync = __napiModule.exports.evaluateUnaryExpressionSync +module.exports.nlEncodeString = __napiModule.exports.nlEncodeString +module.exports.nlTokenizeBatch = __napiModule.exports.nlTokenizeBatch module.exports.overrideConfig = __napiModule.exports.overrideConfig module.exports.renderTemplate = __napiModule.exports.renderTemplate module.exports.renderTemplateSync = __napiModule.exports.renderTemplateSync diff --git a/core/engine/src/policy/blocks/assertion.rs b/core/engine/src/policy/blocks/assertion.rs index 611f7bb9..72a51177 100644 --- a/core/engine/src/policy/blocks/assertion.rs +++ b/core/engine/src/policy/blocks/assertion.rs @@ -2,11 +2,13 @@ use std::sync::Arc; use ahash::HashSet; use serde::{Deserialize, Serialize}; +use zen_expression::intellisense::IntelliSense; use zen_expression::variable::{Variable, VariableType}; use zen_expression::Isolate; use crate::policy::types::{ BlockTrace, ConditionTrace, Cursor, CursorTarget, Diagnostic, DiagnosticCode, ExpressionKind, + NlExpression, }; use crate::policy::ArcStrTrim; @@ -218,6 +220,32 @@ impl AssertionIr { }) } + pub(super) fn nl( + &self, + policy_path: &Arc, + block_id: &Arc, + scope: &VariableType, + is: &mut IntelliSense, + ) -> Vec { + self.conditions + .iter() + .filter(|condition| !condition.expression.is_empty()) + .map(|condition| { + NlExpression::project( + is, + policy_path, + block_id, + CursorTarget::Expression { + id: condition.id.clone(), + }, + ExpressionKind::Standard, + condition.expression.as_ref(), + scope, + ) + }) + .collect() + } + pub(super) fn resolve_cursor( &self, cursor: &Cursor, diff --git a/core/engine/src/policy/blocks/decision_table.rs b/core/engine/src/policy/blocks/decision_table.rs index 1d6ca0ea..3ba25f19 100644 --- a/core/engine/src/policy/blocks/decision_table.rs +++ b/core/engine/src/policy/blocks/decision_table.rs @@ -16,7 +16,7 @@ use base64::Engine as _; use crate::policy::queries::scope::VariableTypeScope; use crate::policy::types::{ BlockTrace, Cursor, CursorTarget, DecisionTableExtras, Diagnostic, DiagnosticCode, - ExpressionKind, + ExpressionKind, NlExpression, }; use crate::policy::ArcStrTrim; @@ -772,6 +772,109 @@ impl DecisionTableIr { self.commit(cx, &selection) } + pub(super) fn nl( + &self, + policy_path: &Arc, + block_id: &Arc, + scope: &VariableType, + is: &mut IntelliSense, + ) -> Vec { + let mut out = Vec::new(); + let mut input_scopes: HashMap, (ExpressionKind, VariableType)> = + HashMap::default(); + + for col in &self.inputs { + match col.field.as_ref().filter(|f| !f.is_empty()) { + Some(field) => { + out.push(NlExpression::project( + is, + policy_path, + block_id, + CursorTarget::DecisionTableHead { + col: col.id.clone(), + }, + ExpressionKind::Standard, + field.as_ref(), + scope, + )); + let field_type = is.analyze(field.as_ref(), scope).return_type.clone(); + input_scopes.insert( + col.id.clone(), + (ExpressionKind::Unary, scope.with_dollar(&field_type)), + ); + } + None => { + input_scopes.insert( + col.id.clone(), + (ExpressionKind::Standard, scope.shallow_clone()), + ); + } + } + } + + for rule in &self.rules { + let Some(row) = rule.get(ROW_ID_KEY) else { + continue; + }; + for col in &self.inputs { + let cell: &str = rule.get(&col.id).map(|c| c.as_ref()).unwrap_or(""); + let Some((kind, cell_scope)) = input_scopes.get(&col.id) else { + continue; + }; + out.push(NlExpression::project( + is, + policy_path, + block_id, + CursorTarget::DecisionTableCell { + row: row.clone(), + col: col.id.clone(), + }, + *kind, + cell, + cell_scope, + )); + } + for col in &self.outputs { + let cell: &str = rule.get(&col.id).map(|c| c.as_ref()).unwrap_or(""); + out.push(NlExpression::project( + is, + policy_path, + block_id, + CursorTarget::DecisionTableCell { + row: row.clone(), + col: col.id.clone(), + }, + ExpressionKind::Standard, + cell, + scope, + )); + } + } + + out + } + + pub(super) fn nl_scope( + &self, + cursor: &Cursor, + scope: VariableType, + is: &mut IntelliSense, + ) -> (ExpressionKind, VariableType) { + let CursorTarget::DecisionTableCell { col, .. } = &cursor.target else { + return (ExpressionKind::Standard, scope); + }; + let Some(ColumnRef::Input(column)) = self.column_by_id(col) else { + return (ExpressionKind::Standard, scope); + }; + match column.field.as_ref().filter(|f| !f.is_empty()) { + Some(field) => { + let field_type = is.analyze(field.as_ref(), &scope).return_type.clone(); + (ExpressionKind::Unary, scope.with_dollar(&field_type)) + } + None => (ExpressionKind::Standard, scope), + } + } + pub(super) fn resolve_cursor( &self, cursor: &Cursor, diff --git a/core/engine/src/policy/blocks/expression.rs b/core/engine/src/policy/blocks/expression.rs index 766e0d7c..4fcfba00 100644 --- a/core/engine/src/policy/blocks/expression.rs +++ b/core/engine/src/policy/blocks/expression.rs @@ -2,10 +2,11 @@ use std::sync::Arc; use ahash::HashSet; use serde::{Deserialize, Serialize}; +use zen_expression::intellisense::IntelliSense; use zen_expression::variable::{Variable, VariableType}; use crate::policy::types::{ - BlockTrace, Cursor, CursorTarget, Diagnostic, DiagnosticCode, ExpressionKind, + BlockTrace, Cursor, CursorTarget, Diagnostic, DiagnosticCode, ExpressionKind, NlExpression, }; use crate::policy::ArcStrTrim; @@ -159,6 +160,29 @@ impl ExpressionIr { }) } + pub(super) fn nl( + &self, + policy_path: &Arc, + block_id: &Arc, + scope: &VariableType, + is: &mut IntelliSense, + ) -> Vec { + if self.value.is_empty() { + return Vec::new(); + } + vec![NlExpression::project( + is, + policy_path, + block_id, + CursorTarget::Expression { + id: self.id.clone(), + }, + ExpressionKind::Standard, + self.value.as_ref(), + scope, + )] + } + pub(super) fn resolve_cursor( &self, cursor: &Cursor, diff --git a/core/engine/src/policy/blocks/match_block.rs b/core/engine/src/policy/blocks/match_block.rs index 9748b689..c947e880 100644 --- a/core/engine/src/policy/blocks/match_block.rs +++ b/core/engine/src/policy/blocks/match_block.rs @@ -2,13 +2,14 @@ use std::sync::Arc; use ahash::HashSet; use serde::{Deserialize, Serialize}; -use zen_expression::intellisense::{ArmTest, NumberCover}; +use zen_expression::intellisense::{ArmTest, IntelliSense, NumberCover}; use zen_expression::variable::{Variable, VariableType}; use crate::policy::queries::scope::VariableTypeScope; use crate::policy::types::{ BlockTrace, ConditionTrace, Cursor, CursorTarget, Diagnostic, DiagnosticCode, ExpressionKind, + NlExpression, }; use crate::policy::ArcStrTrim; @@ -413,6 +414,52 @@ impl MatchIr { self.commit(cx, &selection) } + pub(super) fn nl( + &self, + policy_path: &Arc, + block_id: &Arc, + scope: &VariableType, + is: &mut IntelliSense, + ) -> Vec { + let mut out = Vec::new(); + if !self.key.is_empty() { + out.push(NlExpression::project( + is, + policy_path, + block_id, + CursorTarget::MatchTarget, + ExpressionKind::Standard, + self.key.as_ref(), + scope, + )); + } + for arm in &self.arms { + if !arm.condition.is_empty() { + out.push(NlExpression::project( + is, + policy_path, + block_id, + CursorTarget::Expression { id: arm.id.clone() }, + ExpressionKind::Standard, + arm.condition.as_ref(), + scope, + )); + } + if !arm.value.is_empty() { + out.push(NlExpression::project( + is, + policy_path, + block_id, + CursorTarget::MatchValue { id: arm.id.clone() }, + ExpressionKind::Standard, + arm.value.as_ref(), + scope, + )); + } + } + out + } + pub(super) fn resolve_cursor( &self, cursor: &Cursor, diff --git a/core/engine/src/policy/blocks/mod.rs b/core/engine/src/policy/blocks/mod.rs index 7209a8f3..872e0241 100644 --- a/core/engine/src/policy/blocks/mod.rs +++ b/core/engine/src/policy/blocks/mod.rs @@ -9,11 +9,12 @@ mod type_check; use std::sync::Arc; use ahash::{HashMap, HashMapExt, HashSet}; +use zen_expression::intellisense::IntelliSense; use zen_expression::variable::VariableType; use crate::policy::types::{ BlockTrace, Cursor, CursorTarget, Diagnostic, DiagnosticCode, DiagnosticLocation, - ExpressionKind, Span, + ExpressionKind, NlExpression, Span, }; #[derive(Debug, Clone)] @@ -230,6 +231,27 @@ impl Block { ) -> Option<(Arc, ExpressionKind, VariableType)> { self.kind.resolve_cursor(cursor, scope) } + + pub fn nl( + &self, + policy_path: &Arc, + scope: &VariableType, + is: &mut IntelliSense, + ) -> Vec { + self.kind.nl(policy_path, &self.id, scope, is) + } + + pub fn nl_scope( + &self, + cursor: &Cursor, + scope: VariableType, + is: &mut IntelliSense, + ) -> (ExpressionKind, VariableType) { + match &self.kind { + BlockKind::DecisionTable(d) => d.nl_scope(cursor, scope, is), + _ => (ExpressionKind::Standard, scope), + } + } } impl BlockKind { @@ -319,6 +341,21 @@ impl BlockKind { } } + pub fn nl( + &self, + policy_path: &Arc, + block_id: &Arc, + scope: &VariableType, + is: &mut IntelliSense, + ) -> Vec { + match self { + BlockKind::Assertion(a) => a.nl(policy_path, block_id, scope, is), + BlockKind::DecisionTable(d) => d.nl(policy_path, block_id, scope, is), + BlockKind::Expression(e) => e.nl(policy_path, block_id, scope, is), + BlockKind::Match(m) => m.nl(policy_path, block_id, scope, is), + } + } + pub fn write_keys(&self) -> Vec<(Option>, Arc)> { match self { BlockKind::DecisionTable(d) => d.write_keys(), diff --git a/core/engine/src/policy/editor.rs b/core/engine/src/policy/editor.rs index baf05f4f..86b74e8b 100644 --- a/core/engine/src/policy/editor.rs +++ b/core/engine/src/policy/editor.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use ahash::{HashMap, HashMapExt}; use serde_json::Value; use zen_expression::intellisense::Reference; +use zen_expression::nl::NlResult; use zen_expression::variable::VariableType; use crate::policy::blocks::IntelliSenseSource; @@ -11,7 +12,7 @@ use crate::policy::ir::{DataModelIr, PropertyTypeIr}; use crate::policy::queries::scope::EntityGraph; use crate::policy::types::{ BlockRef, Completion, Cursor, CursorTarget, EngineEdit, ExpressionKind, InspectResult, - PrepareRename, ReferenceKind, ReferenceSite, RenameTarget, Span, SpanOps, + NlExpression, PrepareRename, ReferenceKind, ReferenceSite, RenameTarget, Span, SpanOps, }; impl Db { @@ -37,6 +38,46 @@ impl Db { .completions(&source, cursor.pos, &scope) } + pub fn nl(&self, policy: &str) -> Vec { + let policy_arc: Arc = Arc::from(policy); + let Some(parsed) = self.parsed(&policy_arc) else { + return Vec::new(); + }; + let scope = self.enriched(policy).scope.shallow_clone(); + let intellisense = self.intellisense(); + let mut is = intellisense.borrow_mut(); + let mut out = Vec::new(); + for rule in parsed.policy.rules() { + out.extend(rule.nl(&policy_arc, &scope, &mut is)); + } + out + } + + pub fn nl_tokenize(&self, cursor: &Cursor, text: &str) -> Option { + let (kind, scope) = self.nl_scope(cursor)?; + let unary = matches!(kind, ExpressionKind::Unary); + let intellisense = self.intellisense(); + let mut result = + intellisense + .borrow_mut() + .nl_tokenize_scoped(&cursor.block_id, text, unary, &scope); + if unary { + result.subject_type = Some(scope.get("$")); + } + Some(result) + } + + fn nl_scope(&self, cursor: &Cursor) -> Option<(ExpressionKind, VariableType)> { + let block = self.block_ir(&BlockRef { + policy_path: cursor.policy_path.clone(), + block_id: cursor.block_id.clone(), + })?; + let scope = self.enriched(&cursor.policy_path).scope.shallow_clone(); + let intellisense = self.intellisense(); + let mut is = intellisense.borrow_mut(); + Some(block.nl_scope(cursor, scope, &mut is)) + } + pub fn prepare_rename(&self, cursor: &Cursor) -> Option { if let Some(result) = self.prepare_rename_data_model(cursor) { return Some(result); diff --git a/core/engine/src/policy/mod.rs b/core/engine/src/policy/mod.rs index c0bcdd8b..b323b5a3 100644 --- a/core/engine/src/policy/mod.rs +++ b/core/engine/src/policy/mod.rs @@ -22,9 +22,9 @@ pub use types::{ Diagnostic, DiagnosticCode, DiagnosticLocation, DiscriminantVariant, DiscriminatedUnion, EngineEdit, Entity, EntityField, EvaluateRequest, EvaluationError, EvaluationResult, ExpressionKind, FieldOrigin, GuardedProperty, InputProperty, InputValidationError, - InspectResult, OutputProperty, PrepareRename, PropertyKind, ReferenceKind, ReferenceSite, - RenameTarget, SchemaFieldKind, SchemaGroup, ScopeRequest, Severity, Span, Trace, WriteConflict, - WriteTrace, + InspectResult, NlExpression, OutputProperty, PrepareRename, PropertyKind, ReferenceKind, + ReferenceSite, RenameTarget, SchemaFieldKind, SchemaGroup, ScopeRequest, Severity, Span, Trace, + WriteConflict, WriteTrace, }; pub use workspace::PolicyWorkspace; diff --git a/core/engine/src/policy/types/mod.rs b/core/engine/src/policy/types/mod.rs index 895fa346..dd01d946 100644 --- a/core/engine/src/policy/types/mod.rs +++ b/core/engine/src/policy/types/mod.rs @@ -2,6 +2,7 @@ mod cursor; mod diagnostic; mod edit; mod error; +mod nl; mod request; mod result; @@ -13,6 +14,7 @@ pub(crate) use diagnostic::SpanOps; pub use diagnostic::{Diagnostic, DiagnosticCode, DiagnosticLocation, Severity, Span}; pub use edit::EngineEdit; pub use error::{EvaluationError, InputValidationError}; +pub use nl::NlExpression; pub use request::{EvaluateRequest, ScopeRequest}; pub use result::{ BlockExecution, BlockRef, BlockTrace, Completion, ConditionTrace, ConditionalSchema, diff --git a/core/engine/src/policy/types/nl.rs b/core/engine/src/policy/types/nl.rs new file mode 100644 index 00000000..4a31463b --- /dev/null +++ b/core/engine/src/policy/types/nl.rs @@ -0,0 +1,43 @@ +use std::sync::Arc; + +use zen_expression::intellisense::IntelliSense; +use zen_expression::nl::NlResult; +use zen_expression::variable::VariableType; + +use crate::policy::types::{CursorTarget, ExpressionKind}; + +#[derive(Debug, Clone)] +pub struct NlExpression { + pub policy_path: Arc, + pub block_id: Arc, + pub target: CursorTarget, + pub kind: ExpressionKind, + pub source: String, + pub result: NlResult, +} + +impl NlExpression { + pub(crate) fn project( + is: &mut IntelliSense, + policy_path: &Arc, + block_id: &Arc, + target: CursorTarget, + kind: ExpressionKind, + source: &str, + scope: &VariableType, + ) -> Self { + let unary = matches!(kind, ExpressionKind::Unary); + let mut result = is.nl_tokenize_scoped(block_id, source, unary, scope); + if unary { + result.subject_type = Some(scope.get("$")); + } + Self { + policy_path: policy_path.clone(), + block_id: block_id.clone(), + target, + kind, + source: source.to_string(), + result, + } + } +} diff --git a/core/engine/src/policy/workspace.rs b/core/engine/src/policy/workspace.rs index c47f6f95..727a650d 100644 --- a/core/engine/src/policy/workspace.rs +++ b/core/engine/src/policy/workspace.rs @@ -3,10 +3,13 @@ use std::sync::Arc; use crate::policy::db::Db; use crate::policy::evaluator::EvalArtifact; use crate::policy::raw::PolicyDocument; +use zen_expression::nl::NlResult; + use crate::policy::types::{ Completion, ConditionalSchema, Cursor, DependencyNode, Diagnostic, EngineEdit, Entity, EvaluateRequest, EvaluationError, EvaluationResult, Global, InputProperty, InspectResult, - OutputProperty, PrepareRename, ReferenceSite, RenameTarget, ScopeRequest, WriteConflict, + NlExpression, OutputProperty, PrepareRename, ReferenceSite, RenameTarget, ScopeRequest, + WriteConflict, }; pub struct PolicyWorkspace { @@ -98,6 +101,14 @@ impl PolicyWorkspace { self.db.completions(cursor) } + pub fn nl(&self, policy_path: &str) -> Vec { + self.db.nl(policy_path) + } + + pub fn nl_tokenize(&self, cursor: &Cursor, text: &str) -> Option { + self.db.nl_tokenize(cursor, text) + } + pub fn prepare_rename(&self, cursor: &Cursor) -> Option { self.db.prepare_rename(cursor) } diff --git a/core/engine/tests/policy_nl.rs b/core/engine/tests/policy_nl.rs new file mode 100644 index 00000000..1cff1e30 --- /dev/null +++ b/core/engine/tests/policy_nl.rs @@ -0,0 +1,231 @@ +use serde_json::json; +use zen_engine::policy::{ + Cursor, CursorTarget, ExpressionKind, NlExpression, PolicyDocument, PolicyWorkspace, +}; +use zen_expression::nl::{EditHint, NlTokenKind, OpSym, TypeTag}; + +fn workspace() -> PolicyWorkspace { + let doc: PolicyDocument = serde_json::from_value(json!({ + "blocks": [ + { + "id": "dm", + "type": "dataModel", + "props": { "data": { + "name": "customer", + "properties": [ + { "id": "p1", "name": "age", "type": "number", "array": false, "optional": false }, + { "id": "p2", "name": "tier", "type": "string", "enum": ["gold", "silver", "bronze"], "array": false, "optional": false } + ] + } }, + "children": [] + }, + { + "id": "assert1", + "type": "assertion", + "props": { "data": { + "output": "customer.isAdult", + "conditions": [ + { "id": "c1", "expression": "customer.age >= 18", "operator": "and", "depth": 0 } + ] + } }, + "children": [] + }, + { + "id": "dt1", + "type": "decisionTable", + "props": { "data": { + "hitPolicy": "first", + "inputs": [ + { "id": "in1", "name": "Age", "field": "customer.age" }, + { "id": "in2", "name": "Tier", "field": "customer.tier" } + ], + "outputs": [ { "id": "out1", "name": "Tag", "field": "customer.tag" } ], + "rules": [ { "_id": "row1", "in1": "> 18", "in2": "'gold'", "out1": "'vip'" } ] + } }, + "children": [] + } + ] + })) + .expect("valid policy fixture"); + + let mut ws = PolicyWorkspace::new(); + ws.set_policy("policy", doc); + ws +} + +fn find<'a>(results: &'a [NlExpression], pred: impl Fn(&NlExpression) -> bool) -> &'a NlExpression { + results + .iter() + .find(|e| pred(e)) + .expect("expression present") +} + +fn kinds(e: &NlExpression) -> Vec { + e.result.tokens.iter().map(|t| t.token.clone()).collect() +} + +#[test] +fn assertion_condition_resolves_field_type() { + let ws = workspace(); + let results = ws.nl("policy"); + + let condition = find( + &results, + |e| matches!(&e.target, CursorTarget::Expression { id } if id.as_ref() == "c1"), + ); + + assert_eq!(condition.kind, ExpressionKind::Standard); + assert_eq!(condition.block_id.as_ref(), "assert1"); + assert_eq!( + kinds(condition), + vec![ + NlTokenKind::Field { + path: vec!["customer".into(), "age".into()], + ty: TypeTag::Number, + }, + NlTokenKind::Op { + sym: OpSym::Gte, + implied: false, + between: false, + }, + NlTokenKind::Number { value: "18".into() }, + ] + ); +} + +#[test] +fn decision_table_input_cell_is_unary_with_resolved_subject() { + let ws = workspace(); + let results = ws.nl("policy"); + + let cell = find(&results, |e| { + matches!( + &e.target, + CursorTarget::DecisionTableCell { row, col } + if row.as_ref() == "row1" && col.as_ref() == "in1" + ) + }); + + assert_eq!(cell.kind, ExpressionKind::Unary); + let k = kinds(cell); + assert_eq!( + k[0], + NlTokenKind::Op { + sym: OpSym::Gt, + implied: true, + between: false, + } + ); + assert_eq!(k[1], NlTokenKind::Number { value: "18".into() }); +} + +#[test] +fn decision_table_output_cell_is_standard() { + let ws = workspace(); + let results = ws.nl("policy"); + + let cell = find(&results, |e| { + matches!( + &e.target, + CursorTarget::DecisionTableCell { row, col } + if row.as_ref() == "row1" && col.as_ref() == "out1" + ) + }); + + assert_eq!(cell.kind, ExpressionKind::Standard); + assert!(matches!(kinds(cell).as_slice(), [NlTokenKind::Str { .. }])); +} + +fn cell_cursor(col: &str) -> Cursor { + Cursor { + policy_path: "policy".into(), + block_id: "dt1".into(), + pos: 0, + target: CursorTarget::DecisionTableCell { + row: "row1".into(), + col: col.into(), + }, + } +} + +#[test] +fn nl_tokenize_resolves_unary_enum_subject() { + let ws = workspace(); + let result = ws + .nl_tokenize(&cell_cursor("in2"), "'gold'") + .expect("cursor resolves"); + + let str_tok = result + .tokens + .iter() + .find(|t| matches!(t.token, NlTokenKind::Str { .. })) + .expect("string token present"); + assert_eq!(str_tok.hint, Some(EditHint::Select { options: 0 })); + let labels: Vec<&str> = result.enums[0].iter().map(|o| o.label.as_str()).collect(); + assert_eq!(labels, vec!["gold", "silver", "bronze"]); + assert!(result.enums[0] + .iter() + .all(|o| o.source.as_deref() == Some(format!("\"{}\"", o.label).as_str()))); +} + +#[test] +fn nl_tokenize_projects_live_text_over_stored_cell() { + let ws = workspace(); + let result = ws + .nl_tokenize(&cell_cursor("in1"), "< 21") + .expect("cursor resolves"); + + let tokens: Vec = result.tokens.iter().map(|t| t.token.clone()).collect(); + assert_eq!( + tokens[0], + NlTokenKind::Op { + sym: OpSym::Lt, + implied: true, + between: false, + } + ); + assert_eq!(tokens[1], NlTokenKind::Number { value: "21".into() }); +} + +#[test] +fn nl_tokenize_standard_expression_uses_policy_scope() { + let ws = workspace(); + let cursor = Cursor { + policy_path: "policy".into(), + block_id: "assert1".into(), + pos: 0, + target: CursorTarget::Expression { id: "c1".into() }, + }; + let result = ws + .nl_tokenize(&cursor, "customer.age >= 21") + .expect("cursor resolves"); + + let tokens: Vec = result.tokens.iter().map(|t| t.token.clone()).collect(); + assert_eq!( + tokens[0], + NlTokenKind::Field { + path: vec!["customer".into(), "age".into()], + ty: TypeTag::Number, + } + ); +} + +#[test] +fn decision_table_input_head_is_projected() { + let ws = workspace(); + let results = ws.nl("policy"); + + let head = find( + &results, + |e| matches!(&e.target, CursorTarget::DecisionTableHead { col } if col.as_ref() == "in1"), + ); + + assert_eq!(head.kind, ExpressionKind::Standard); + assert_eq!( + kinds(head), + vec![NlTokenKind::Field { + path: vec!["customer".into(), "age".into()], + ty: TypeTag::Number, + }] + ); +} diff --git a/core/expression/src/intellisense/mod.rs b/core/expression/src/intellisense/mod.rs index 922267be..9c9b8888 100644 --- a/core/expression/src/intellisense/mod.rs +++ b/core/expression/src/intellisense/mod.rs @@ -9,6 +9,8 @@ use crate::intellisense::inspection::{inspect_at, InspectionResult}; use crate::intellisense::scope::IntelliSenseScope; use crate::intellisense::type_provider::TypesProvider; use crate::lexer::Lexer; +use crate::nl::project::Projector; +use crate::nl::{NlRequest, NlResult}; use crate::parser::{Node, NodeMetadata, Parser}; use crate::variable::VariableType; use bumpalo::Bump; @@ -181,6 +183,118 @@ impl IntelliSense { } } + pub fn nl_tokenize_batch( + &mut self, + requests: &[NlRequest], + root_type: &VariableType, + ) -> Vec { + requests + .iter() + .map(|request| self.nl_tokenize(request, root_type)) + .collect() + } + + pub fn nl_tokenize(&mut self, request: &NlRequest, root_type: &VariableType) -> NlResult { + let scope = if request.unary { + Self::unary_scope(root_type, request.subject_type.as_ref()) + } else { + root_type.shallow_clone() + }; + let mut result = + self.nl_tokenize_scoped(&request.id, &request.expression, request.unary, &scope); + if request.unary { + result.subject_type = Some(scope.get("$")); + } + result + } + + pub fn nl_tokenize_scoped( + &mut self, + id: &str, + source: &str, + unary: bool, + scope_type: &VariableType, + ) -> NlResult { + let mut result = NlResult { + id: id.to_string(), + tokens: Vec::new(), + enums: Vec::new(), + diagnostics: Vec::new(), + subject_type: None, + }; + + self.arena.reset(); + let arena = &self.arena; + + let tokens = match self.lexer.tokenize(arena, source) { + Ok(tokens) => tokens, + Err(err) => { + result.diagnostics.push(lexer_error_to_diagnostic(&err)); + return result; + } + }; + + let Ok(parser) = Parser::try_new(&tokens, arena) else { + return result; + }; + + let parser_result = if unary { + parser.unary().with_metadata().parse() + } else { + parser.standard().with_metadata().parse() + }; + let ast = parser_result.root; + + if !parser_result.is_complete || ast.has_error() { + if !parser_result.is_complete { + result.diagnostics.push(Diagnostic { + span: (0, 0), + message: "Incomplete expression".to_string(), + severity: Severity::Error, + source: DiagnosticSource::Parser, + }); + } + collect_parser_diagnostics(ast, &mut result.diagnostics); + return result; + } + + let metadata = parser_result.metadata.unwrap_or_default(); + + let scope = IntelliSenseScope { + pointer_data: scope_type.shallow_clone(), + root_data: scope_type.shallow_clone(), + current_data: scope_type.shallow_clone(), + ..Default::default() + }; + + let type_data = TypesProvider::generate(ast, scope, self.strict); + collect_type_diagnostics(ast, &type_data, &metadata, &mut result.diagnostics); + + let (tokens, enums) = Projector::new(source, &type_data, &metadata, unary).run(ast); + result.tokens = tokens; + result.enums = enums; + result + } + + fn unary_scope(root_type: &VariableType, subject_type: Option<&VariableType>) -> VariableType { + let subject = subject_type + .map(|s| s.shallow_clone()) + .unwrap_or(VariableType::Any); + + let object = VariableType::empty_object(); + if let VariableType::Object(target) = &object { + if let VariableType::Object(source) = root_type { + for (key, value) in source.borrow().iter() { + target + .borrow_mut() + .insert(key.clone(), value.shallow_clone()); + } + } + target.borrow_mut().insert(Rc::from("$"), subject); + } + object + } + pub fn with_ast( &mut self, source: &str, diff --git a/core/expression/src/lib.rs b/core/expression/src/lib.rs index f630ec5c..2ea10125 100644 --- a/core/expression/src/lib.rs +++ b/core/expression/src/lib.rs @@ -65,6 +65,7 @@ pub mod expression; pub mod functions; pub mod intellisense; pub mod lexer; +pub mod nl; pub mod parser; pub mod validate; pub mod variable; diff --git a/core/expression/src/nl/mod.rs b/core/expression/src/nl/mod.rs new file mode 100644 index 00000000..fd5ed933 --- /dev/null +++ b/core/expression/src/nl/mod.rs @@ -0,0 +1,38 @@ +pub(crate) mod project; +pub mod token; + +pub use token::{EditHint, EnumOption, NlToken, NlTokenKind, OpChoice, OpSym, TypeTag, WordSym}; + +use serde::Serialize; + +use crate::intellisense::diagnostic::Diagnostic; +use crate::variable::VariableType; + +pub fn encode_string(value: &str) -> Option { + if !value.contains('"') { + Some(format!("\"{value}\"")) + } else if !value.contains('\'') { + Some(format!("'{value}'")) + } else { + None + } +} + +#[derive(Debug, Clone)] +pub struct NlRequest { + pub id: String, + pub expression: String, + pub unary: bool, + pub subject_type: Option, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct NlResult { + pub id: String, + pub tokens: Vec, + pub enums: Vec>, + pub diagnostics: Vec, + #[serde(skip)] + pub subject_type: Option, +} diff --git a/core/expression/src/nl/project.rs b/core/expression/src/nl/project.rs new file mode 100644 index 00000000..de43b656 --- /dev/null +++ b/core/expression/src/nl/project.rs @@ -0,0 +1,696 @@ +use std::rc::Rc; + +use crate::functions::FunctionKind; +use crate::intellisense::type_provider::TypesProvider; +use crate::intellisense::AstMetadata; +use crate::lexer::{Bracket, Operator}; +use crate::nl::encode_string; +use crate::nl::token::{ + EditHint, EnumOption, NlToken, NlTokenKind, OpChoice, OpSym, TypeTag, WordSym, +}; +use crate::parser::Node; +use crate::variable::VariableType; + +struct AliasScope { + name: Option>, + elide: bool, +} + +pub(crate) struct Projector<'a> { + source: &'a str, + types: &'a TypesProvider, + metadata: &'a AstMetadata, + unary: bool, + aliases: Vec, + pending_elide: bool, + pending_implied: bool, + enums: Vec>, + out: Vec, +} + +impl<'a> Projector<'a> { + pub(crate) fn new( + source: &'a str, + types: &'a TypesProvider, + metadata: &'a AstMetadata, + unary: bool, + ) -> Self { + Self { + source, + types, + metadata, + unary, + aliases: Vec::new(), + pending_elide: false, + pending_implied: false, + enums: Vec::new(), + out: Vec::new(), + } + } + + pub(crate) fn run(mut self, root: &Node) -> (Vec, Vec>) { + self.project(root, None); + (self.out, self.enums) + } + + fn project(&mut self, node: &Node, expected: Option) { + let span = self.span_of(node); + + match node { + Node::Null => self.push(NlTokenKind::Null, span), + Node::Bool(value) => self.push(NlTokenKind::Bool { value: *value }, span), + Node::Number(value) => self.push( + NlTokenKind::Number { + value: value.normalize().to_string().into_boxed_str(), + }, + span, + ), + Node::String(value) => { + let hint = match Self::enum_values(expected.as_ref()) { + Some(values) => Some(EditHint::Select { + options: self.intern_enum(&values), + }), + None if Self::expects_date(expected.as_ref()) => Some(EditHint::DatePicker), + None => None, + }; + self.push_hint( + NlTokenKind::Str { + value: Box::from(*value), + }, + span, + hint, + ) + } + + Node::TemplateString(parts) => { + self.push(NlTokenKind::TemplateOpen, (span.0, span.0)); + for part in parts.iter() { + match part { + Node::String(value) => self.push( + NlTokenKind::TemplateText { + value: Box::from(*value), + }, + self.span_of(part), + ), + other => self.project(other, None), + } + } + self.push(NlTokenKind::TemplateClose, (span.1, span.1)); + } + + Node::Root => self.push(NlTokenKind::Root, span), + Node::Pointer => self.push(NlTokenKind::Element { alias: None }, span), + + Node::Identifier(name) => { + if *name == "$" { + self.push(NlTokenKind::Context, span); + } else if self + .aliases + .iter() + .any(|scope| scope.name.as_deref() == Some(*name)) + { + self.push( + NlTokenKind::Element { + alias: Some(Box::from(*name)), + }, + span, + ); + } else { + let ty = self.tag_of(&self.type_of(node)); + self.push( + NlTokenKind::Field { + path: vec![Box::from(*name)], + ty, + }, + span, + ); + } + } + + Node::Member { .. } => match Self::field_path(node) { + Some(field) => { + let field = self.strip_elided_alias(field); + let ty = self.tag_of(&self.type_of(node)); + self.push(NlTokenKind::Field { path: field, ty }, span); + } + None => self.code(span), + }, + + Node::Binary { + left, + operator, + right, + } => { + let (exp_left, exp_right) = self.operand_expectations(*operator, left, right); + let context_subject = matches!(operator, Operator::Comparison(_)) + && matches!(left, Node::Identifier(name) if *name == "$"); + if !context_subject && !self.is_elided_subject(left) { + self.project(left, exp_left); + } + if let Some(sym) = OpSym::from_operator(*operator) { + let op_span = (self.span_of(left).1, self.span_of(right).0); + let hint = self.op_hint(*operator, left); + let implied = context_subject || std::mem::take(&mut self.pending_implied); + let between = matches!(sym, OpSym::In | OpSym::NotIn) + && matches!(right, Node::Interval { .. }); + self.push_hint( + NlTokenKind::Op { + sym, + implied, + between, + }, + op_span, + hint, + ); + } + self.project(right, exp_right); + } + + Node::Unary { + node: inner, + operator, + } => { + if let Some(sym) = OpSym::from_operator(*operator) { + let implied = std::mem::take(&mut self.pending_implied); + self.push( + NlTokenKind::Op { + sym, + implied, + between: false, + }, + (span.0, self.span_of(inner).0), + ); + } + self.project(inner, None); + } + + Node::Conditional { + condition, + on_true, + on_false, + } => { + self.push(NlTokenKind::Word { sym: WordSym::If }, (span.0, span.0)); + self.project(condition, None); + self.push( + NlTokenKind::Word { sym: WordSym::Then }, + (self.span_of(condition).1, self.span_of(on_true).0), + ); + self.project(on_true, None); + self.push( + NlTokenKind::Word { + sym: WordSym::Otherwise, + }, + (self.span_of(on_true).1, self.span_of(on_false).0), + ); + self.project(on_false, None); + } + + Node::Interval { + left, + right, + left_bracket, + right_bracket, + } => { + self.push( + NlTokenKind::IntervalOpen { + inclusive: *left_bracket == Bracket::LeftSquareBracket, + }, + (span.0, span.0), + ); + self.project(left, None); + self.push( + NlTokenKind::Word { + sym: WordSym::RangeAnd, + }, + (self.span_of(left).1, self.span_of(right).0), + ); + self.project(right, None); + self.push( + NlTokenKind::IntervalClose { + inclusive: *right_bracket == Bracket::RightSquareBracket, + }, + (span.1, span.1), + ); + } + + Node::Parenthesized(inner) => { + self.push(NlTokenKind::GroupOpen, (span.0, span.0)); + self.project(inner, expected); + self.push(NlTokenKind::GroupClose, (span.1, span.1)); + } + + Node::Array(items) => { + let enum_domain = Self::enum_values(expected.as_ref()) + .filter(|_| items.iter().all(|item| matches!(item, Node::String(_)))); + if let Some(values) = enum_domain { + let selected = items + .iter() + .filter_map(|item| match item { + Node::String(value) => Some(Box::from(*value)), + _ => None, + }) + .collect(); + let hint = EditHint::MultiSelect { + options: self.intern_enum(&values), + }; + self.push_hint(NlTokenKind::EnumList { selected }, span, Some(hint)); + return; + } + self.push(NlTokenKind::ListOpen, (span.0, span.0)); + let item_expected = Self::item_expectation(expected.as_ref()); + for (i, item) in items.iter().enumerate() { + if i > 0 { + let prev = self.span_of(items[i - 1]); + self.push(NlTokenKind::Comma, (prev.1, self.span_of(item).0)); + } + self.project(item, item_expected.clone()); + } + self.push(NlTokenKind::ListClose, (span.1, span.1)); + } + + Node::FunctionCall { kind, arguments } => self.function_call(kind, arguments, span), + + Node::MethodCall { + kind, + this, + arguments, + } => { + self.project(this, None); + let this_end = self.span_of(this).1; + self.push( + NlTokenKind::Method { + sym: kind.to_string().into_boxed_str(), + }, + (this_end, span.1), + ); + if let [only] = arguments { + if Self::is_simple_arg(only) { + self.project(only, None); + return; + } + } + self.group_args(arguments, this_end, span.1); + } + + Node::Closure { body, alias } => { + let elide = std::mem::take(&mut self.pending_elide); + self.aliases.push(AliasScope { + name: alias.map(Box::from), + elide, + }); + self.project(body, expected); + self.aliases.pop(); + } + + Node::Assignments { list, output } => { + for (i, (key, value)) in list.iter().enumerate() { + if i > 0 { + let prev_end = self.span_of(list[i - 1].1).1; + self.push(NlTokenKind::StmtEnd, (prev_end, self.span_of(key).0)); + } + match key { + Node::Identifier(name) | Node::String(name) => { + let ty = self.tag_of(&self.type_of(value)); + self.push( + NlTokenKind::Field { + path: vec![Box::from(*name)], + ty, + }, + self.span_of(key), + ); + } + other => self.project(other, None), + } + self.push( + NlTokenKind::Assign, + (self.span_of(key).1, self.span_of(value).0), + ); + self.project(value, None); + } + if let Some(output) = output { + if let Some(last) = list.last() { + self.push( + NlTokenKind::StmtEnd, + (self.span_of(last.1).1, self.span_of(output).0), + ); + } + self.project(output, expected); + } + } + + Node::Object(_) | Node::Slice { .. } | Node::Error { .. } => self.code(span), + } + } + + fn function_call(&mut self, kind: &FunctionKind, arguments: &[&Node], span: (u32, u32)) { + let sym = kind.to_string().into_boxed_str(); + let FunctionKind::Closure(_) = kind else { + if Self::is_infix_predicate(&sym) && arguments.len() == 2 { + let (left, right) = (arguments[0], arguments[1]); + self.project(left, None); + self.push( + NlTokenKind::Func { + sym, + closure: false, + }, + (self.span_of(left).1, self.span_of(right).0), + ); + self.project(right, None); + return; + } + if !(self.unary && sym.as_ref() == "bool" && self.out.is_empty()) { + self.push( + NlTokenKind::Func { + sym, + closure: false, + }, + (span.0, span.0), + ); + } + if let [only] = arguments { + if Self::is_simple_arg(only) { + self.project(only, None); + return; + } + } + self.group_args(arguments, span.0, span.1); + return; + }; + + self.push(NlTokenKind::Func { sym, closure: true }, (span.0, span.0)); + + let alias = match arguments.get(1) { + Some(Node::Closure { alias, .. }) => *alias, + _ => None, + }; + let elide = self.aliases.is_empty(); + if !elide { + self.push( + NlTokenKind::Element { + alias: alias.map(Box::from), + }, + (span.0, span.0), + ); + self.push(NlTokenKind::Word { sym: WordSym::In }, (span.0, span.0)); + } + if let Some(collection) = arguments.first() { + self.project(collection, None); + } + let leftmost = match arguments.get(1) { + Some(Node::Closure { body, .. }) => Some(Self::leftmost_leaf(body)), + _ => None, + }; + let body_leads_with_subject = + elide && leftmost.is_some_and(|leaf| Self::is_binding_leaf(leaf, alias)); + if !body_leads_with_subject { + let sym = if elide && leftmost.is_some_and(|leaf| Self::is_binding_member(leaf, alias)) + { + self.pending_implied = true; + WordSym::Has + } else { + WordSym::Where + }; + self.push(NlTokenKind::Word { sym }, (span.1, span.1)); + } + if let Some(closure) = arguments.get(1) { + self.pending_elide = elide; + self.project(closure, None); + } + } + + fn is_infix_predicate(sym: &str) -> bool { + matches!( + sym, + "contains" | "startsWith" | "endsWith" | "matches" | "fuzzyMatch" + ) + } + + fn is_simple_arg(node: &Node) -> bool { + match node { + Node::Identifier(_) + | Node::Number(_) + | Node::String(_) + | Node::Bool(_) + | Node::Pointer + | Node::Array(_) => true, + Node::Member { .. } => Self::field_path(node).is_some(), + _ => false, + } + } + + fn leftmost_leaf<'n>(body: &'n Node<'n>) -> &'n Node<'n> { + let mut node = body; + loop { + match node { + Node::Binary { left, .. } => node = left, + Node::Parenthesized(inner) => node = inner, + other => return other, + } + } + } + + fn is_binding_leaf(leaf: &Node, alias: Option<&str>) -> bool { + match leaf { + Node::Pointer => alias.is_none(), + Node::Identifier(name) => alias == Some(*name), + _ => false, + } + } + + fn is_binding_member(leaf: &Node, alias: Option<&str>) -> bool { + match leaf { + Node::Member { .. } => match Self::field_path(leaf) { + Some(field) => match (field.first(), alias) { + (Some(head), Some(alias)) => head.as_ref() == alias, + (Some(head), None) => head.as_ref() == "#", + _ => false, + }, + None => false, + }, + _ => false, + } + } + + fn alias_elided(&self, name: Option<&str>) -> bool { + match name { + None => self + .aliases + .last() + .is_some_and(|scope| scope.elide && scope.name.is_none()), + Some(name) => self + .aliases + .iter() + .rev() + .find(|scope| scope.name.as_deref() == Some(name)) + .is_some_and(|scope| scope.elide), + } + } + + fn is_elided_subject(&self, node: &Node) -> bool { + match node { + Node::Pointer => self.alias_elided(None), + Node::Identifier(name) => self.alias_elided(Some(name)), + _ => false, + } + } + + fn strip_elided_alias(&self, field: Vec>) -> Vec> { + if field.len() < 2 { + return field; + } + let head = field[0].as_ref(); + let elided = self.alias_elided((head != "#").then_some(head)); + if elided { + field.into_iter().skip(1).collect() + } else { + field + } + } + + fn group_args(&mut self, arguments: &[&Node], open: u32, close: u32) { + self.push(NlTokenKind::GroupOpen, (open, open)); + for (i, arg) in arguments.iter().enumerate() { + if i > 0 { + let prev = self.span_of(arguments[i - 1]); + self.push(NlTokenKind::Comma, (prev.1, self.span_of(arg).0)); + } + self.project(arg, None); + } + self.push(NlTokenKind::GroupClose, (close, close)); + } + + fn op_hint(&self, operator: Operator, left: &Node) -> Option { + use crate::lexer::ComparisonOperator as Cmp; + use crate::lexer::LogicalOperator as Log; + let options = match operator { + Operator::Comparison(Cmp::In | Cmp::NotIn) => vec![OpSym::In, OpSym::NotIn], + Operator::Comparison(_) => { + if Self::is_ordered(&self.type_of(left)) { + vec![ + OpSym::Gt, + OpSym::Gte, + OpSym::Lt, + OpSym::Lte, + OpSym::Eq, + OpSym::Ne, + ] + } else { + vec![OpSym::Eq, OpSym::Ne] + } + } + Operator::Logical(Log::And | Log::Or) => vec![OpSym::And, OpSym::Or], + Operator::Arithmetic(_) => vec![OpSym::Add, OpSym::Sub, OpSym::Mul, OpSym::Div], + _ => return None, + }; + Some(EditHint::OpSelect { + options: options.into_iter().map(OpChoice::from).collect(), + }) + } + + fn is_ordered(ty: &VariableType) -> bool { + match ty { + VariableType::Number | VariableType::Date | VariableType::Any => true, + VariableType::Nullable(inner) => Self::is_ordered(inner), + _ => false, + } + } + + fn operand_expectations( + &self, + operator: Operator, + left: &Node, + right: &Node, + ) -> (Option, Option) { + use crate::lexer::ComparisonOperator::{In, NotIn}; + let Operator::Comparison(comparison) = operator else { + return (None, None); + }; + match comparison { + In | NotIn => (None, Some(self.type_of(left))), + _ => (Some(self.type_of(right)), Some(self.type_of(left))), + } + } + + fn code(&mut self, span: (u32, u32)) { + let source = self + .source + .get(span.0 as usize..span.1 as usize) + .unwrap_or_default(); + self.push( + NlTokenKind::Code { + source: Box::from(source), + }, + span, + ); + } + + fn push(&mut self, token: NlTokenKind, span: (u32, u32)) { + self.push_hint(token, span, None); + } + + fn push_hint(&mut self, token: NlTokenKind, span: (u32, u32), hint: Option) { + self.out.push(NlToken { token, span, hint }); + } + + fn span_of(&self, node: &Node) -> (u32, u32) { + let addr = node as *const Node as usize; + node.span() + .or_else(|| self.metadata.get(&addr).map(|m| m.span)) + .unwrap_or_default() + } + + fn type_of(&self, node: &Node) -> VariableType { + self.types + .get_type(node) + .map(|t| t.kind.clone()) + .unwrap_or(VariableType::Any) + } + + fn tag_of(&mut self, ty: &VariableType) -> TypeTag { + match ty { + VariableType::Number => TypeTag::Number, + VariableType::String | VariableType::Const(_) => TypeTag::String, + VariableType::Bool => TypeTag::Bool, + VariableType::Date => TypeTag::Date, + VariableType::Interval => TypeTag::Interval, + VariableType::Object(_) => TypeTag::Object, + VariableType::Null => TypeTag::Null, + VariableType::Any => TypeTag::Unknown, + VariableType::Enum(_, values) => TypeTag::Enum { + index: self.intern_enum(values), + }, + VariableType::Array(inner) => TypeTag::Array { + items: Box::new(self.tag_of(inner)), + }, + VariableType::Nullable(inner) => self.tag_of(inner), + } + } + + fn intern_enum(&mut self, values: &[Rc]) -> u32 { + let existing = self.enums.iter().position(|e| { + e.len() == values.len() && e.iter().zip(values).all(|(a, b)| a.label == b.as_ref()) + }); + if let Some(index) = existing { + return index as u32; + } + self.enums.push( + values + .iter() + .map(|v| EnumOption { + label: v.to_string(), + source: encode_string(v), + }) + .collect(), + ); + (self.enums.len() - 1) as u32 + } + + fn enum_values(ty: Option<&VariableType>) -> Option>> { + match ty? { + VariableType::Enum(_, values) => Some(values.clone()), + VariableType::Nullable(inner) | VariableType::Array(inner) => { + Self::enum_values(Some(inner)) + } + _ => None, + } + } + + fn expects_date(ty: Option<&VariableType>) -> bool { + match ty { + Some(VariableType::Date) => true, + Some(VariableType::Nullable(inner)) => Self::expects_date(Some(inner)), + _ => false, + } + } + + fn item_expectation(ty: Option<&VariableType>) -> Option { + match ty? { + VariableType::Array(inner) | VariableType::Nullable(inner) => { + Self::item_expectation(Some(inner)) + } + other => Some(other.shallow_clone()), + } + } + + fn field_path(node: &Node) -> Option>> { + match node { + Node::Identifier(name) => Some(vec![Box::from(*name)]), + Node::Pointer => Some(vec![Box::from("#")]), + Node::Root => Some(vec![Box::from("$root")]), + Node::Member { node, property } => { + let mut base = Self::field_path(node)?; + match property { + Node::String(s) => base.push(Box::from(*s)), + Node::Root => base.push(Box::from("$root")), + Node::Number(n) if n.is_integer() && !n.is_sign_negative() => { + let last = base.pop()?; + base.push(Box::from(format!("{last}[{n}]").as_str())); + } + _ => return None, + } + Some(base) + } + _ => None, + } + } +} diff --git a/core/expression/src/nl/token.rs b/core/expression/src/nl/token.rs new file mode 100644 index 00000000..c17c4c6f --- /dev/null +++ b/core/expression/src/nl/token.rs @@ -0,0 +1,221 @@ +use serde::Serialize; + +use crate::lexer::{ArithmeticOperator, ComparisonOperator, LogicalOperator, Operator}; + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct NlToken { + pub token: NlTokenKind, + pub span: (u32, u32), + #[serde(skip_serializing_if = "Option::is_none")] + pub hint: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(tag = "t", rename_all = "camelCase")] +pub enum NlTokenKind { + GroupOpen, + GroupClose, + ListOpen, + ListClose, + Comma, + EnumList { + selected: Vec>, + }, + + Context, + Root, + Null, + + Field { + path: Vec>, + ty: TypeTag, + }, + Element { + #[serde(skip_serializing_if = "Option::is_none")] + alias: Option>, + }, + + Number { + value: Box, + }, + Str { + value: Box, + }, + Bool { + value: bool, + }, + + Op { + sym: OpSym, + implied: bool, + between: bool, + }, + Word { + sym: WordSym, + }, + Assign, + StmtEnd, + Func { + sym: Box, + closure: bool, + }, + Method { + sym: Box, + }, + + TemplateOpen, + TemplateText { + value: Box, + }, + TemplateClose, + + IntervalOpen { + inclusive: bool, + }, + IntervalClose { + inclusive: bool, + }, + + Code { + source: Box, + }, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(tag = "t", rename_all = "camelCase")] +pub enum TypeTag { + Number, + String, + Bool, + Date, + Interval, + Object, + Null, + Unknown, + Enum { index: u32 }, + Array { items: Box }, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(tag = "kind", rename_all = "camelCase")] +pub enum EditHint { + DatePicker, + Select { options: u32 }, + MultiSelect { options: u32 }, + OpSelect { options: Vec }, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OpChoice { + pub sym: OpSym, + pub source: &'static str, +} + +impl From for OpChoice { + fn from(sym: OpSym) -> Self { + Self { + sym, + source: sym.source(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct EnumOption { + pub label: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub source: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum OpSym { + Gt, + Gte, + Lt, + Lte, + Eq, + Ne, + In, + NotIn, + Add, + Sub, + Mul, + Div, + Mod, + Pow, + And, + Or, + Not, + Coalesce, +} + +impl OpSym { + pub fn source(&self) -> &'static str { + match self { + OpSym::Gt => ">", + OpSym::Gte => ">=", + OpSym::Lt => "<", + OpSym::Lte => "<=", + OpSym::Eq => "==", + OpSym::Ne => "!=", + OpSym::In => "in", + OpSym::NotIn => "not in", + OpSym::Add => "+", + OpSym::Sub => "-", + OpSym::Mul => "*", + OpSym::Div => "/", + OpSym::Mod => "%", + OpSym::Pow => "^", + OpSym::And => "and", + OpSym::Or => "or", + OpSym::Not => "not", + OpSym::Coalesce => "??", + } + } + + pub(crate) fn from_operator(operator: Operator) -> Option { + match operator { + Operator::Arithmetic(a) => Some(match a { + ArithmeticOperator::Add => OpSym::Add, + ArithmeticOperator::Subtract => OpSym::Sub, + ArithmeticOperator::Multiply => OpSym::Mul, + ArithmeticOperator::Divide => OpSym::Div, + ArithmeticOperator::Modulus => OpSym::Mod, + ArithmeticOperator::Power => OpSym::Pow, + }), + Operator::Logical(l) => Some(match l { + LogicalOperator::And => OpSym::And, + LogicalOperator::Or => OpSym::Or, + LogicalOperator::Not => OpSym::Not, + LogicalOperator::NullishCoalescing => OpSym::Coalesce, + }), + Operator::Comparison(c) => Some(match c { + ComparisonOperator::Equal => OpSym::Eq, + ComparisonOperator::NotEqual => OpSym::Ne, + ComparisonOperator::LessThan => OpSym::Lt, + ComparisonOperator::GreaterThan => OpSym::Gt, + ComparisonOperator::LessThanOrEqual => OpSym::Lte, + ComparisonOperator::GreaterThanOrEqual => OpSym::Gte, + ComparisonOperator::In => OpSym::In, + ComparisonOperator::NotIn => OpSym::NotIn, + }), + _ => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum WordSym { + If, + Then, + Otherwise, + In, + Where, + Has, + RangeAnd, +} diff --git a/core/expression/tests/nl.rs b/core/expression/tests/nl.rs new file mode 100644 index 00000000..ae70106b --- /dev/null +++ b/core/expression/tests/nl.rs @@ -0,0 +1,715 @@ +use std::rc::Rc; + +use zen_expression::intellisense::IntelliSense; +use zen_expression::nl::{ + encode_string, EditHint, EnumOption, NlRequest, NlResult, NlTokenKind, OpChoice, OpSym, + TypeTag, WordSym, +}; +use zen_expression::variable::VariableType; + +fn op(sym: OpSym) -> NlTokenKind { + NlTokenKind::Op { + sym, + implied: false, + between: false, + } +} + +fn op_implied(sym: OpSym) -> NlTokenKind { + NlTokenKind::Op { + sym, + implied: true, + between: false, + } +} + +fn choices(syms: &[OpSym]) -> Vec { + syms.iter().copied().map(OpChoice::from).collect() +} + +fn obj(fields: &[(&str, VariableType)]) -> VariableType { + let object = VariableType::empty_object(); + if let VariableType::Object(map) = &object { + for (key, value) in fields { + map.borrow_mut().insert(Rc::from(*key), value.clone()); + } + } + object +} + +fn enum_t(name: &str, values: &[&str]) -> VariableType { + VariableType::Enum( + Some(Rc::from(name)), + values.iter().map(|v| Rc::from(*v)).collect(), + ) +} + +fn array(inner: VariableType) -> VariableType { + VariableType::Array(Rc::new(inner)) +} + +fn run(expr: &str, unary: bool, subject: Option, root: &VariableType) -> NlResult { + let mut intellisense = IntelliSense::new(); + intellisense.nl_tokenize( + &NlRequest { + id: "x".into(), + expression: expr.into(), + unary, + subject_type: subject, + }, + root, + ) +} + +fn kinds(result: &NlResult) -> Vec { + result.tokens.iter().map(|t| t.token.clone()).collect() +} + +#[test] +fn comparison_number() { + let root = obj(&[("customer", obj(&[("age", VariableType::Number)]))]); + let result = run("customer.age >= 18", false, None, &root); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + assert_eq!( + kinds(&result), + vec![ + NlTokenKind::Field { + path: vec!["customer".into(), "age".into()], + ty: TypeTag::Number, + }, + op(OpSym::Gte), + NlTokenKind::Number { value: "18".into() }, + ] + ); + + let number = result + .tokens + .iter() + .find(|t| matches!(t.token, NlTokenKind::Number { .. })) + .unwrap(); + assert_eq!(number.hint, None); +} + +#[test] +fn enum_membership_multiselect() { + let root = obj(&[("tier", enum_t("Tier", &["gold", "silver", "bronze"]))]); + let result = run("tier in ['gold', 'silver']", false, None, &root); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + + let k = kinds(&result); + assert_eq!( + k[0], + NlTokenKind::Field { + path: vec!["tier".into()], + ty: TypeTag::Enum { index: 0 }, + } + ); + assert_eq!(k[1], op(OpSym::In)); + assert_eq!( + k[2], + NlTokenKind::EnumList { + selected: vec!["gold".into(), "silver".into()], + } + ); + assert_eq!(k.len(), 3); + assert_eq!( + result.tokens[2].hint, + Some(EditHint::MultiSelect { options: 0 }) + ); + + assert_eq!( + result.enums, + vec![vec![ + EnumOption { + label: "gold".to_string(), + source: Some("\"gold\"".to_string()), + }, + EnumOption { + label: "silver".to_string(), + source: Some("\"silver\"".to_string()), + }, + EnumOption { + label: "bronze".to_string(), + source: Some("\"bronze\"".to_string()), + }, + ]] + ); +} + +#[test] +fn enum_list_spans_whole_array() { + let root = obj(&[("tier", enum_t("Tier", &["gold", "silver"]))]); + let result = run("tier in ['gold']", false, None, &root); + + let list = result + .tokens + .iter() + .find(|t| matches!(t.token, NlTokenKind::EnumList { .. })) + .unwrap(); + let source = "tier in ['gold']"; + assert_eq!( + &source[list.span.0 as usize..list.span.1 as usize], + "['gold']" + ); +} + +#[test] +fn mixed_enum_array_stays_plain_list() { + let root = obj(&[ + ("tier", enum_t("Tier", &["gold", "silver"])), + ("other", VariableType::String), + ]); + let result = run("tier in ['gold', other]", false, None, &root); + + let k = kinds(&result); + assert!(k.contains(&NlTokenKind::ListOpen)); + assert!(k.contains(&NlTokenKind::ListClose)); + assert!(!k.iter().any(|t| matches!(t, NlTokenKind::EnumList { .. }))); +} + +#[test] +fn encode_string_swaps_quotes() { + assert_eq!(encode_string("plain"), Some("\"plain\"".to_string())); + assert_eq!(encode_string("6\" nail"), Some("'6\" nail'".to_string())); + assert_eq!(encode_string("it's"), Some("\"it's\"".to_string())); + assert_eq!(encode_string("both \" and '"), None); +} + +#[test] +fn closure_elides_top_level_alias() { + let root = obj(&[("countries", array(enum_t("Country", &["US", "CA", "GB"])))]); + let result = run("all(countries as c, c == 'US')", false, None, &root); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + + let k = kinds(&result); + assert_eq!( + k[0], + NlTokenKind::Func { + sym: "all".into(), + closure: true, + } + ); + assert!(matches!(&k[1], NlTokenKind::Field { .. })); + assert_eq!(k[2], op(OpSym::Eq)); + assert!(matches!(&k[3], NlTokenKind::Str { .. })); +} + +#[test] +fn closure_elides_alias_in_member_paths() { + let root = obj(&[( + "order", + obj(&[("items", array(obj(&[("price", VariableType::Number)])))]), + )]); + let result = run( + "all(order.items as item, item.price > 100)", + false, + None, + &root, + ); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + + let k = kinds(&result); + assert_eq!( + k[0], + NlTokenKind::Func { + sym: "all".into(), + closure: true, + } + ); + assert!( + matches!(&k[1], NlTokenKind::Field { path, .. } if path.len() == 2 && path[0].as_ref() == "order") + ); + assert_eq!(k[2], NlTokenKind::Word { sym: WordSym::Has }); + assert!( + matches!(&k[3], NlTokenKind::Field { path, .. } if path.len() == 1 && path[0].as_ref() == "price") + ); + assert_eq!(k[4], op_implied(OpSym::Gt)); +} + +#[test] +fn closure_callback_reference() { + let root = obj(&[("orders", array(VariableType::Number))]); + let result = run("some(orders, # > 100)", false, None, &root); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + + let k = kinds(&result); + assert_eq!( + k[0], + NlTokenKind::Func { + sym: "some".into(), + closure: true, + } + ); + assert!(matches!(&k[1], NlTokenKind::Field { .. })); + assert_eq!(k[2], op(OpSym::Gt)); +} + +#[test] +fn unary_number_cell() { + let result = run("> 18", true, Some(VariableType::Number), &VariableType::Any); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + + let k = kinds(&result); + assert_eq!(k[0], op_implied(OpSym::Gt)); + assert_eq!(k[1], NlTokenKind::Number { value: "18".into() }); +} + +#[test] +fn unary_enum_cell() { + let subject = enum_t("Tier", &["gold", "silver"]); + let result = run("'gold'", true, Some(subject), &VariableType::Any); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + + let k = kinds(&result); + assert_eq!(k[0], op_implied(OpSym::Eq)); + assert!(matches!(&k[1], NlTokenKind::Str { .. })); + assert_eq!(result.tokens[1].hint, Some(EditHint::Select { options: 0 })); +} + +#[test] +fn function_call_projects_args() { + let root = obj(&[("scores", array(VariableType::Number))]); + let result = run("sum(scores)", false, None, &root); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + + let k = kinds(&result); + assert_eq!( + k[0], + NlTokenKind::Func { + sym: "sum".into(), + closure: false, + } + ); + assert!(matches!(&k[1], NlTokenKind::Field { .. })); + assert_eq!(k.len(), 2); +} + +#[test] +fn assignments_project_statements() { + let root = obj(&[("base", VariableType::Number)]); + let result = run("x = base * 2; y = x + 1; y > 10", false, None, &root); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + let k = kinds(&result); + let tags: Vec<&str> = k + .iter() + .map(|t| match t { + NlTokenKind::Field { .. } => "field", + NlTokenKind::Assign => "assign", + NlTokenKind::StmtEnd => "stmtEnd", + NlTokenKind::Op { .. } => "op", + NlTokenKind::Number { .. } => "number", + other => panic!("unexpected token {other:?}"), + }) + .collect(); + assert_eq!( + tags, + vec![ + "field", "assign", "field", "op", "number", "stmtEnd", "field", "assign", "field", + "op", "number", "stmtEnd", "field", "op", "number", + ] + ); + assert_eq!( + k[0], + NlTokenKind::Field { + path: vec!["x".into()], + ty: TypeTag::Number, + } + ); +} + +#[test] +fn constant_index_member_projects_as_field() { + let root = obj(&[( + "revenue", + obj(&[( + "tiers", + array(obj(&[("itdReceipts", VariableType::Number)])), + )]), + )]); + let result = run("revenue.tiers[0].itdReceipts > 0", false, None, &root); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + assert_eq!( + kinds(&result), + vec![ + NlTokenKind::Field { + path: vec!["revenue".into(), "tiers[0]".into(), "itdReceipts".into()], + ty: TypeTag::Number, + }, + op(OpSym::Gt), + NlTokenKind::Number { value: "0".into() }, + ] + ); +} + +#[test] +fn dynamic_index_member_stays_code() { + let root = obj(&[ + ("items", array(VariableType::Number)), + ("i", VariableType::Number), + ]); + let result = run("items[i]", false, None, &root); + + let k = kinds(&result); + assert_eq!( + k[0], + NlTokenKind::Code { + source: "items[i]".into(), + } + ); +} + +#[test] +fn slice_falls_back_to_code() { + let root = obj(&[("items", array(VariableType::Number))]); + let result = run("items[1:3]", false, None, &root); + + let k = kinds(&result); + assert_eq!( + k[0], + NlTokenKind::Code { + source: "items[1:3]".into(), + } + ); +} + +#[test] +fn unknown_function_half_stays_structured() { + let root = obj(&[("name", VariableType::String)]); + let result = run("fuzzyMatch(name, 'jon') > 0.8", false, None, &root); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + + let k = kinds(&result); + assert!(matches!(&k[0], NlTokenKind::Field { .. })); + assert_eq!( + k[1], + NlTokenKind::Func { + sym: "fuzzyMatch".into(), + closure: false, + } + ); + assert!(matches!(&k[2], NlTokenKind::Str { .. })); + assert!(k.iter().any(|t| *t == op(OpSym::Gt))); +} + +#[test] +fn date_comparison_hints_date_picker() { + let root = obj(&[("start", VariableType::Date)]); + let result = run("start >= '2026-01-01'", false, None, &root); + + let date = result + .tokens + .iter() + .find(|t| matches!(t.token, NlTokenKind::Str { .. })) + .unwrap(); + assert_eq!(date.hint, Some(EditHint::DatePicker)); +} + +#[test] +fn conditional_projects_words() { + let root = obj(&[("age", VariableType::Number)]); + let result = run("age > 18 ? 'adult' : 'minor'", false, None, &root); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + + let k = kinds(&result); + assert_eq!(k[0], NlTokenKind::Word { sym: WordSym::If }); + assert!(matches!(&k[1], NlTokenKind::Field { .. })); + assert_eq!(k[2], op(OpSym::Gt)); + assert!(matches!(&k[3], NlTokenKind::Number { .. })); + assert_eq!(k[4], NlTokenKind::Word { sym: WordSym::Then }); + assert!(matches!(&k[5], NlTokenKind::Str { .. })); + assert_eq!( + k[6], + NlTokenKind::Word { + sym: WordSym::Otherwise, + } + ); + assert!(matches!(&k[7], NlTokenKind::Str { .. })); +} + +#[test] +fn template_string_projects_parts() { + let root = obj(&[("name", VariableType::String)]); + let result = run("`Hi ${name}!`", false, None, &root); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + + let k = kinds(&result); + assert_eq!(k[0], NlTokenKind::TemplateOpen); + assert_eq!( + k[1], + NlTokenKind::TemplateText { + value: "Hi ".into(), + } + ); + assert_eq!( + k[2], + NlTokenKind::Field { + path: vec!["name".into()], + ty: TypeTag::String, + } + ); + assert_eq!(k[3], NlTokenKind::TemplateText { value: "!".into() }); + assert_eq!(k[4], NlTokenKind::TemplateClose); +} + +#[test] +fn unary_interval_projects_range() { + let result = run( + "[18..30)", + true, + Some(VariableType::Number), + &VariableType::Any, + ); + + let k = kinds(&result); + assert!(k.contains(&NlTokenKind::IntervalOpen { inclusive: true })); + assert!(k.contains(&NlTokenKind::Word { + sym: WordSym::RangeAnd, + })); + assert!(k.contains(&NlTokenKind::IntervalClose { inclusive: false })); +} + +#[test] +fn membership_over_interval_marks_between() { + let root = obj(&[("age", VariableType::Number)]); + let result = run("age in [18..65]", false, None, &root); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + let k = kinds(&result); + assert!(k.contains(&NlTokenKind::Op { + sym: OpSym::In, + implied: false, + between: true, + })); +} + +#[test] +fn op_choices_carry_source() { + let root = obj(&[("tier", enum_t("Tier", &["gold", "silver"]))]); + let result = run("tier != 'gold'", false, None, &root); + + let op_tok = result + .tokens + .iter() + .find(|t| matches!(t.token, NlTokenKind::Op { .. })) + .unwrap(); + let Some(EditHint::OpSelect { options }) = &op_tok.hint else { + panic!("expected opSelect hint"); + }; + let sources: Vec<&str> = options.iter().map(|o| o.source).collect(); + assert_eq!(sources, vec!["==", "!="]); +} + +#[test] +fn method_call_projects_receiver_and_args() { + let root = obj(&[("created", VariableType::Date)]); + let result = run("created.format('yyyy-MM-dd')", false, None, &root); + + let k = kinds(&result); + assert!(matches!(&k[0], NlTokenKind::Field { .. })); + assert_eq!( + k[1], + NlTokenKind::Method { + sym: "format".into(), + } + ); + assert!(matches!(&k[2], NlTokenKind::Str { .. })); + assert_eq!(k.len(), 3); +} + +#[test] +fn nested_closures_shadow_alias() { + let root = obj(&[( + "teams", + array(obj(&[("members", array(VariableType::String))])), + )]); + let result = run( + "some(teams as t, some(t.members as m, m == 'lee'))", + false, + None, + &root, + ); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + + let k = kinds(&result); + let elements: Vec<_> = k + .iter() + .filter_map(|t| match t { + NlTokenKind::Element { alias } => Some(alias.clone()), + _ => None, + }) + .collect(); + assert_eq!(elements, vec![Some("m".into()), Some("m".into())]); + assert!(!k + .iter() + .any(|t| matches!(t, NlTokenKind::Field { path, .. } if path == &vec![Box::from("m")]))); +} + +#[test] +fn json_wire_shape() { + let root = obj(&[("customer", obj(&[("age", VariableType::Number)]))]); + let result = run("customer.age >= 18", false, None, &root); + + let value = serde_json::to_value(&result).unwrap(); + let tokens = value["tokens"].as_array().unwrap(); + + assert_eq!(tokens[0]["token"]["t"], "field"); + assert_eq!(tokens[0]["token"]["path"][0], "customer"); + assert_eq!(tokens[0]["token"]["ty"]["t"], "number"); + assert_eq!(tokens[1]["token"]["t"], "op"); + assert_eq!(tokens[1]["token"]["sym"], "gte"); + assert_eq!(tokens[2]["token"]["t"], "number"); + assert_eq!(tokens[2]["token"]["value"], "18"); + assert_eq!(tokens[2].get("hint"), None); +} + +#[test] +fn batch_correlates_ids() { + let root = obj(&[("age", VariableType::Number)]); + let mut intellisense = IntelliSense::new(); + let requests = vec![ + NlRequest { + id: "a".into(), + expression: "age > 1".into(), + unary: false, + subject_type: None, + }, + NlRequest { + id: "b".into(), + expression: "age < 5".into(), + unary: false, + subject_type: None, + }, + ]; + + let results = intellisense.nl_tokenize_batch(&requests, &root); + assert_eq!(results.len(), 2); + assert_eq!(results[0].id, "a"); + assert_eq!(results[1].id, "b"); +} + +#[test] +fn op_hint_ordered_for_numbers() { + let root = obj(&[("customer", obj(&[("age", VariableType::Number)]))]); + let result = run("customer.age >= 18", false, None, &root); + + let op = result + .tokens + .iter() + .find(|t| matches!(t.token, NlTokenKind::Op { .. })) + .unwrap(); + assert_eq!( + op.hint, + Some(EditHint::OpSelect { + options: choices(&[ + OpSym::Gt, + OpSym::Gte, + OpSym::Lt, + OpSym::Lte, + OpSym::Eq, + OpSym::Ne, + ]) + }) + ); +} + +#[test] +fn op_hint_equality_only_for_enums() { + let root = obj(&[( + "customer", + obj(&[("tier", enum_t("Tier", &["gold", "silver"]))]), + )]); + let result = run("customer.tier == 'gold'", false, None, &root); + + let op = result + .tokens + .iter() + .find(|t| matches!(t.token, NlTokenKind::Op { .. })) + .unwrap(); + assert_eq!( + op.hint, + Some(EditHint::OpSelect { + options: choices(&[OpSym::Eq, OpSym::Ne]) + }) + ); +} + +#[test] +fn op_hint_membership_and_joiners() { + let root = obj(&[ + ("age", VariableType::Number), + ("tier", enum_t("Tier", &["gold", "silver"])), + ]); + let result = run("age > 18 and tier in ['gold']", false, None, &root); + + let ops: Vec<_> = result + .tokens + .iter() + .filter(|t| matches!(t.token, NlTokenKind::Op { .. })) + .collect(); + assert_eq!(ops.len(), 3); + assert_eq!( + ops[1].hint, + Some(EditHint::OpSelect { + options: choices(&[OpSym::And, OpSym::Or]) + }) + ); + assert_eq!( + ops[2].hint, + Some(EditHint::OpSelect { + options: choices(&[OpSym::In, OpSym::NotIn]) + }) + ); +} + +#[test] +fn infix_predicate_functions() { + let root = obj(&[("name", VariableType::String)]); + let result = run("contains(name, 'jo')", false, None, &root); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + + let k = kinds(&result); + assert!(matches!(&k[0], NlTokenKind::Field { .. })); + assert_eq!( + k[1], + NlTokenKind::Func { + sym: "contains".into(), + closure: false, + } + ); + assert!(matches!(&k[2], NlTokenKind::Str { .. })); + assert_eq!(k.len(), 3); +} + +#[test] +fn complex_single_arg_keeps_group() { + let root = obj(&[("total", VariableType::Number)]); + let result = run("round(total * 0.1)", false, None, &root); + + assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics); + + let k = kinds(&result); + assert_eq!( + k[0], + NlTokenKind::Func { + sym: "round".into(), + closure: false, + } + ); + assert_eq!(k[1], NlTokenKind::GroupOpen); + assert_eq!(*k.last().unwrap(), NlTokenKind::GroupClose); +}