c api: enable JIT from embedded hosts; fix the macOS in-memory JIT arm

doc/source/reference/embedding/c_api.rst

@@ -185,6 +185,13 @@ File access
 ``das_fileaccess_introduce_native_module(access, req)`` / ``das_fileaccess_introduce_native_module_n(access, req, req_length)``
    Pre-loads one native module by its require-style path.
 
+``das_fileaccess_add_extra_module(access, module_name, module_file)`` / ``das_fileaccess_add_extra_module_n(...)``
+   Registers a module as an implicit dependency of every program compiled
+   through this file access, as if each program had a ``require`` for it.
+   This is how the daslang CLI injects daslib support modules:
+   ``just_in_time`` (``-jit``), ``debug`` (``-debugger``), ``profiler``
+   (``-profiler``).
+
 ``das_fileaccess_lock(access)`` / ``das_fileaccess_unlock(access)``
    While locked, only pre-introduced files can be accessed — filesystem
    reads are blocked.  Essential for sandboxing.
@@ -195,6 +202,13 @@ File access
 ``das_get_root_n(buf, maxbuf)``
    ``size_t``-sized counterpart.  Passing ``NULL, 0`` is valid.
 
+``das_set_root(root)`` / ``das_set_root_n(root, root_length)``
+   Sets the daslang root path.  When unset, the root is derived from the
+   host executable's location (walking up past ``bin/``), which is wrong
+   for embedded applications that live outside the daslang distribution —
+   call this before creating file access objects so ``daslib/``,
+   ``modules/`` and ``lib/`` resolve.
+
 
 Compilation
 ===========
@@ -291,6 +305,14 @@ Compilation policies
      - Generate extended RTTI
    * - ``DAS_POLICY_NO_OPTIMIZATIONS``
      - Disable all optimizations
+   * - ``DAS_POLICY_JIT_ENABLED``
+     - Enable JIT compilation (see *Enabling JIT* below)
+   * - ``DAS_POLICY_JIT_DLL_MODE``
+     - With JIT: cache generated code as per-script DLLs under
+       ``<dasroot>/.jitted_scripts`` (default: on); off = in-memory codegen
+       every run, no cache writes.  The cache path needs a DLL build of
+       daslang — a static-linked host always codegens in-memory regardless
+       of this flag
 
 **Integer policies** (``das_int_policy``):
 
@@ -313,6 +335,38 @@ Compilation policies
 See :ref:`tutorial_integration_c_sandbox`.
 
 
+Enabling JIT
+------------
+
+The CLI's ``-jit`` switch is three host-side steps; an embedded host mirrors
+them through the C API.  The daslang root must point at a distribution that
+carries ``daslib/``, ``modules/dasLLVM`` and ``lib/`` (the LLVM backend
+library) — set it explicitly when the host executable lives elsewhere.
+Script-side ``options jit_enabled = true`` has no effect: JIT is a host
+policy.
+
+.. code-block:: c
+
+   das_set_root("/path/to/daslang");           /* unless the exe sits in <root>/bin */
+
+   das_file_access * fa = das_fileaccess_make_default();
+   das_register_dynamic_modules(fa, "/path/to/daslang", NULL, 0, tout);
+
+   char root[4096], jit_mod[4200];
+   das_get_root_n(root, sizeof(root));
+   snprintf(jit_mod, sizeof(jit_mod), "%s/daslib/just_in_time.das", root);
+   das_fileaccess_add_extra_module(fa, "just_in_time", jit_mod);
+
+   das_policies * pol = das_policies_make();
+   das_policies_set_bool(pol, DAS_POLICY_JIT_ENABLED, 1);
+
+   das_program * prog = das_program_compile_policies(
+       "script.das", fa, tout, lib, pol);
+
+After ``das_program_simulate()`` the ``[jit]`` functions run natively;
+``jit_enabled()`` returns ``true`` inside the script.
+
+
 Simulation and context
 ======================
 

include/daScript/daScriptC.h

@@ -267,6 +267,17 @@ DAS_CC_API void das_fileaccess_introduce_native_modules ( das_file_access * acce
 DAS_CC_API int das_fileaccess_introduce_native_module ( das_file_access * access, const char * req );
 DAS_CC_API int das_fileaccess_introduce_native_module_n ( das_file_access * access, const char * req, size_t req_length );
 
+// Register a module as an implicit dependency of every program compiled through
+// this file access, as if each program had a 'require' for it. This is how the
+// daslang CLI injects daslib support modules: "just_in_time" ('-jit'),
+// "debug" ('-debugger'), "profiler" ('-profiler').
+// 'module_name' is the require name; 'module_file' is the path to its source
+// (e.g. "<dasroot>/daslib/just_in_time.das").
+DAS_CC_API void das_fileaccess_add_extra_module ( das_file_access * access, const char * module_name, const char * module_file );
+DAS_CC_API void das_fileaccess_add_extra_module_n ( das_file_access * access,
+                                                    const char * module_name, size_t module_name_length,
+                                                    const char * module_file, size_t module_file_length );
+
 // Lock the file access. While locked, getNewFileInfo() returns NULL for all
 // files not already in the cache, so only pre-introduced files can be accessed.
 DAS_CC_API void das_fileaccess_lock ( das_file_access * access );
@@ -280,6 +291,13 @@ DAS_CC_API int das_fileaccess_is_locked ( das_file_access * access );
 DAS_CC_API void das_get_root ( char * root, int maxbuf );
 DAS_CC_API void das_get_root_n ( char * root, size_t maxbuf );
 
+// Set the daslang root path. When unset, the root is derived from the host
+// executable's location (walking up past bin/), which is wrong for embedded
+// applications that live outside the daslang distribution - call this before
+// creating file access objects so daslib/, modules/ and lib/ resolve.
+DAS_CC_API void das_set_root ( const char * root );
+DAS_CC_API void das_set_root_n ( const char * root, size_t root_length );
+
 // --- Compilation ---
 
 // Compile a daslang program from the file at 'program_file'.
@@ -787,7 +805,9 @@ typedef enum das_bool_policy {
     DAS_POLICY_LOG_OPTIMIZATION_PASSES,          // Log the AST after every optimizer pass (verbose)
     DAS_POLICY_FUSION,                           // Fuse interpreter nodes into wider superinstructions at simulate time (default: on)
     DAS_POLICY_AUTO_INLINE_FUNCTIONS,            // Heuristic best-effort inlining of plain calls and operator sites of small same-module [inline]-shaped functions (default ON; silent declines; optimized builds only; cross-module inlining stays the explicit [inline] contract)
-    DAS_POLICY_DISABLE_TEMP_STRING_RECLAIM       // Disable the temp-string reclaim pass (fresh-string call results riding the 1-slot dispose queue)
+    DAS_POLICY_DISABLE_TEMP_STRING_RECLAIM,      // Disable the temp-string reclaim pass (fresh-string call results riding the 1-slot dispose queue)
+    DAS_POLICY_JIT_ENABLED,                      // Enable JIT compilation of [jit] functions. The flag alone is not enough: the host must also register the "just_in_time" extra module (see das_fileaccess_add_extra_module) so the LLVM JIT backend is compiled in
+    DAS_POLICY_JIT_DLL_MODE                      // With JIT: cache generated code as per-script DLLs under <dasroot>/.jitted_scripts (default: on). Off = codegen in-memory every run, no cache writes. The cache path needs a DLL build of daslang; a static-linked host always codegens in-memory regardless of this flag
 } das_bool_policy;
 
 // Integer policy fields (stack size, heap limits).

include/daScript/simulate/aot_builtin_ast.h

@@ -453,6 +453,8 @@ namespace das {
     DAS_CC_API void addModuleLintMacro ( Module * module, PassMacroPtr _newM, Context * );
     DAS_CC_API void addModuleGlobalLintMacro ( Module * module, PassMacroPtr _newM, Context * );
     DAS_CC_API void addModuleOptimizationMacro ( Module * module, PassMacroPtr _newM, Context * );
+    DAS_CC_API void addModulePostRewriteMacro ( Module * module, PassMacroPtr _newM, Context * );
+    DAS_CC_API void addModulePostCompileMacro ( Module * module, PassMacroPtr _newM, Context * );
     DAS_CC_API VariantMacroPtr makeVariantMacro ( const char * name, const void * pClass, const StructInfo * info, Context * context );
     DAS_CC_API void addModuleVariantMacro ( Module * module, VariantMacroPtr newM, Context * context );
     DAS_CC_API ForLoopMacroPtr makeForLoopMacro ( const char * name, const void * pClass, const StructInfo * info, Context * context );

modules/dasGlsl/CMakeLists.txt

@@ -5,6 +5,12 @@ IF(NOT DAS_GLSL_INCLUDED)
     ADD_MODULE_DAS_FROM_DESCRIPTOR(glsl glsl)
     ADD_MODULE_DAS_FROM_DESCRIPTOR(geometry glsl)
 
+    # runtime-callable module code the AOT suite must carry stubs for (glsl_* stays out:
+    # emission-side macro code, nothing calls it at runtime)
+    SET(DASGLSL_AOT_FILES
+        modules/dasGlsl/glsl/geom_gen.das
+)
+
     install(DIRECTORY ${PROJECT_SOURCE_DIR}/modules/dasGlsl/glsl
         DESTINATION ${DAS_INSTALL_MODULESDIR}/dasGlsl
         FILES_MATCHING

modules/dasLLVM/daslib/llvm_jit.das

@@ -7583,7 +7583,7 @@ def private get_llvm_function_type(func : FunctionPtr; types : PrimitiveTypes?)
 }
 
 [macro_function]
-def private add_ctor_dtor_function(ctor_dtor : array<tuple<LLVMOpaqueValue?, LLVMOpaqueValue?>>; types : PrimitiveTypes?) {
+def private add_ctor_dtor_function(ctor_dtor : array<tuple<LLVMOpaqueValue?, LLVMOpaqueValue?>>; types : PrimitiveTypes?; skip_dtors : bool) {
     let priority = 0 |> uint64()
     let dataPointerType = LLVMPointerType(types.t_int8, 0u)
     let funcType = LLVMPointerType(LLVMFunctionType(types.t_void, null, 0u, 0), 0u)
@@ -7635,7 +7635,7 @@ def private add_ctor_dtor_function(ctor_dtor : array<tuple<LLVMOpaqueValue?, LLV
         LLVMSetLinkage(ctorArray, LLVMLinkage.LLVMAppendingLinkage)
         LLVMSetInitializer(ctorArray, LLVMConstArray(ctorStructType, array_data_ptr(ctor_entries), ctor_sz))
     }
-    if (dtor_sz != 0u) {
+    if (dtor_sz != 0u && !skip_dtors) {
         let dtorArray = LLVMAddGlobal(g_mod, LLVMArrayType(dtorStructType, dtor_sz), "llvm.global_dtors")
         LLVMSetLinkage(dtorArray, LLVMLinkage.LLVMAppendingLinkage)
         LLVMSetInitializer(dtorArray, LLVMConstArray(dtorStructType, array_data_ptr(dtor_entries), dtor_sz))
@@ -8113,14 +8113,24 @@ def public generate_llvm_code(var dll : DLLHandle?; fmna : DllName; dll_enabled
 // optional pre-built pair (the LLVM-AOT self-registration ctor, built by llvm_aot). aot_ctor==null skips it.
 [macro_function]
 def public generate_global_ctors_dtors(types : PrimitiveTypes?; jit_mode : bool; aot_ctor : LLVMOpaqueValue? = null; aot_dtor : LLVMOpaqueValue? = null) {
+    // nolint:STYLE014 - the failure is silent at both ends; both halves of the WHY must survive
+    // Mach-O codegen lowers llvm.global_dtors into __cxa_atexit(.., extern_weak @__dso_handle)
+    // registrations, and an in-memory image can satisfy neither end: RuntimeDyld's failed
+    // __dso_handle lookup silently aborts external resolution for the WHOLE module (every
+    // extern's GOT slot stays null; the first ctor jumps to 0), while a successful
+    // registration would fire at process exit() into the by-then-disposed engine's memory.
+    // So the darwin in-memory arm emits no dtor list at all - dll/exe arms are real images
+    // and keep it. (If another lowering ever references __dso_handle, a defined hidden
+    // dummy global by that name keeps resolution self-contained.)
+    let in_memory_darwin = jit_mode && get_platform_name() == "darwin"
     var ctor_dtor <- [
         // FileInfo
         generate_fileinfo_ctor_dtor(*types, active_filenames, jit_mode)
     ]
     if (aot_ctor != null) {
         ctor_dtor |> push((aot_ctor, aot_dtor))
     }
-    ctor_dtor |> add_ctor_dtor_function(types)
+    ctor_dtor |> add_ctor_dtor_function(types, in_memory_darwin)
     reset_codegen_accumulators()
 }
 

modules/dasLLVM/daslib/llvm_jit_run.das

@@ -36,11 +36,11 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole
 // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das,
 // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump
 // makes every previously written DLL miss the cache on the next run and get GC'd.
-let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x55ul   // handled types iterate and index through the annotation's jit hooks (0x54: indirect calls to jitted functions inline the dispatcher, cmres included)
+let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x56ul   // darwin in-memory arm emits no dtor list (0x55: handled types iterate and index through the annotation's jit hooks)
 
 // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources
 // (normalized to LF; file list in the test)
-let LLVM_JIT_EMITTER_HASH : uint64 = 0xebb45abe203cad70ul
+let LLVM_JIT_EMITTER_HASH : uint64 = 0x3140f6c8e0a8738eul
 
 let JIT_FNV_PRIME : uint64 = 1099511628211ul
 

src/misc/daScriptC.cpp

@@ -374,6 +374,20 @@ int das_fileaccess_introduce_native_module_n ( das_file_access * access, const c
     return ((FsFileAccess *) access)->introduceNativeModule(req_string) ? 1 : 0;
 }
 
+void das_fileaccess_add_extra_module ( das_file_access * access, const char * module_name, const char * module_file ) {
+    das_fileaccess_add_extra_module_n(access,
+        module_name, c_string_length(module_name, "module_name"),
+        module_file, c_string_length(module_file, "module_file"));
+}
+
+void das_fileaccess_add_extra_module_n ( das_file_access * access,
+                                         const char * module_name, size_t module_name_length,
+                                         const char * module_file, size_t module_file_length ) {
+    auto module_name_string = string_from_range(module_name, module_name_length, "module_name");
+    auto module_file_string = string_from_range(module_file, module_file_length, "module_file");
+    ((FileAccess *) access)->addExtraModule(module_name_string, module_file_string);
+}
+
 void das_fileaccess_lock ( das_file_access * access ) {
     ((FileAccess *) access)->lock();
 }
@@ -404,6 +418,14 @@ void das_get_root_n ( char * root, size_t maxbuf ) {
     root[copy_length] = 0;
 }
 
+void das_set_root ( const char * root ) {
+    das_set_root_n(root, c_string_length(root, "root"));
+}
+
+void das_set_root_n ( const char * root, size_t root_length ) {
+    setDasRoot(string_from_range(root, root_length, "root"));
+}
+
 das_program * das_program_compile ( char * program_file, das_file_access * access, das_text_writer * tout, das_module_group * libgroup ) {
     return das_program_compile_n(program_file, c_string_length(program_file, "program_file"), access, tout, libgroup);
 }
@@ -925,6 +947,8 @@ int das_policies_set_bool ( das_policies * policies, das_bool_policy flag, int v
         case DAS_POLICY_LOG_OPTIMIZATION_PASSES: p->log_optimization_passes = v; break;
         case DAS_POLICY_FUSION:                 p->fusion = v; break;
         case DAS_POLICY_AUTO_INLINE_FUNCTIONS:  p->auto_inline_functions = v; break;
+        case DAS_POLICY_JIT_ENABLED:            p->jit_enabled = v; break;
+        case DAS_POLICY_JIT_DLL_MODE:           p->jit_dll_mode = v; break;
         default: return 0;
     }
     return 1;

src/misc/sysos.cpp

@@ -661,6 +661,8 @@ namespace das {
 
     void setDasRoot ( const string & dr ) {
         g_dasRoot = dr;
+        // same normalization the derived root gets below - callers concatenate with '/'
+        replace(g_dasRoot.begin(), g_dasRoot.end(), '\\', '/');
     }
 
     string getDasRoot ( void ) {

tests-cpp/small/test_capi_jit.cpp

@@ -0,0 +1,126 @@
+#include <doctest/doctest.h>
+
+#include "daScript/daScriptC.h"
+
+#include <cstdio>
+#include <string>
+
+namespace {
+
+struct CApiProgram {
+    das_text_writer * output = nullptr;
+    das_module_group * modules = nullptr;
+    das_file_access * files = nullptr;
+    das_policies * policies = nullptr;
+    das_program * program = nullptr;
+    das_context * context = nullptr;
+
+    ~CApiProgram() {
+        if ( context ) das_context_release(context);
+        if ( program ) das_program_release(program);
+        if ( policies ) das_policies_release(policies);
+        if ( files ) das_fileaccess_release(files);
+        if ( modules ) das_modulegroup_release(modules);
+        if ( output ) das_text_release(output);
+    }
+};
+
+// the script asserts jit_enabled() and that the [jit] function actually went
+// through the JIT — a panic surfaces as das_context_get_exception()
+const char * JIT_SCRIPT =
+    "options gen2\n"
+    "[jit]\n"
+    "def sq(x : int) {\n"
+    "    return x * x\n"
+    "}\n"
+    "[export]\n"
+    "def main {\n"
+    "    if (!jit_enabled()) {\n"
+    "        panic(\"jit_enabled() is false\")\n"
+    "    }\n"
+    "    if (!is_jit_function(@@sq)) {\n"
+    "        panic(\"sq was not JIT compiled\")\n"
+    "    }\n"
+    "    if (sq(7) != 49) {\n"
+    "        panic(\"sq(7) != 49\")\n"
+    "    }\n"
+    "}\n";
+
+bool fileExists ( const std::string & path ) {
+    if ( FILE * f = fopen(path.c_str(), "rb") ) {
+        fclose(f);
+        return true;
+    }
+    return false;
+}
+
+}
+
+TEST_CASE("C API set root round trip") {
+    char root[4096];
+    das_get_root(root, int(sizeof(root)));
+    std::string dasRoot = root;
+    REQUIRE(!dasRoot.empty());
+    // a root DIFFERENT from the derived one, set through the explicit-length form
+    // with a non-NUL-terminated range - a no-op das_set_root fails both checks
+    const char newRoot[] = {'/','n','o','/','s','u','c','h','/','r','o','o','t','X'};
+    das_set_root_n(newRoot, sizeof(newRoot) - 1);
+    char root2[4096];
+    das_get_root(root2, int(sizeof(root2)));
+    CHECK(std::string(root2) == "/no/such/root");
+    das_set_root(dasRoot.c_str());
+    das_get_root(root2, int(sizeof(root2)));
+    CHECK(dasRoot == root2);
+}
+
+TEST_CASE("C API jit enable") {
+    char root[4096];
+    das_get_root(root, int(sizeof(root)));
+    std::string dasRoot = root;
+    if ( !fileExists(dasRoot + "/lib/LLVM.dll") ) {
+        MESSAGE("no lib/LLVM.dll in this build tree - skipping the JIT C API test");
+        return;
+    }
+
+    CApiProgram capi;
+    capi.output = das_text_make_writer();
+    capi.modules = das_modulegroup_make();
+    capi.files = das_fileaccess_make_default();
+    capi.policies = das_policies_make();
+
+    // the embedded mirror of the CLI's -jit switch: the policy flag, the
+    // just_in_time extra module, and dynamic-module resolution for dasLLVM
+    REQUIRE(das_policies_set_bool(capi.policies, DAS_POLICY_JIT_ENABLED, 1) == 1);
+    // off = the in-memory arm on every build flavor - the path the darwin fix targets
+    REQUIRE(das_policies_set_bool(capi.policies, DAS_POLICY_JIT_DLL_MODE, 0) == 1);
+    REQUIRE(das_register_dynamic_modules(capi.files, dasRoot.c_str(), nullptr, 0, capi.output) == 0);
+    std::string jitModule = dasRoot + "/daslib/just_in_time.das";
+    das_fileaccess_add_extra_module(capi.files, "just_in_time", jitModule.c_str());
+
+    das_fileaccess_introduce_file(capi.files, "jit_script.das", JIT_SCRIPT, 0);
+    const char scriptName[] = "jit_script.das";
+    capi.program = das_program_compile_policies_n(scriptName, sizeof(scriptName) - 1,
+        capi.files, capi.output, capi.modules, capi.policies);
+    REQUIRE(capi.program != nullptr);
+    if ( das_program_err_count(capi.program) != 0 ) {
+        for ( int i = 0; i != das_program_err_count(capi.program); ++i ) {
+            das_error * err = das_program_get_error(capi.program, i);
+            char buf[2048];
+            das_error_report(err, buf, int(sizeof(buf)));
+            MESSAGE(buf);
+        }
+        FAIL("compilation failed");
+    }
+
+    capi.context = das_context_make(das_program_context_stack_size(capi.program));
+    REQUIRE(capi.context != nullptr);
+    REQUIRE(das_program_simulate(capi.program, capi.context, capi.output) == 1);
+
+    das_function * fnMain = das_context_find_function(capi.context, "main");
+    REQUIRE(fnMain != nullptr);
+    das_context_eval_with_catch(capi.context, fnMain, nullptr);
+    char * exception = das_context_get_exception(capi.context);
+    CHECK_MESSAGE(exception == nullptr, std::string(exception ? exception : ""));
+}
+// a regression of the darwin dtor-skip shows up as SIGSEGV at process exit() -
+// AFTER doctest prints SUCCESS; the exit code is the signal, and ctest reads it