diff --git a/README.md b/README.md index c81bd8870..6d9a2f897 100644 --- a/README.md +++ b/README.md @@ -182,8 +182,6 @@ profiler: address: localhost:6060 lua: - proto_cache_size: 60000 - main_cache_size: 10000 type_system: enabled: true strict: false @@ -191,6 +189,9 @@ lua: enabled: true dir: .wippy/cache/lua mode: readwrite # off | readonly | readwrite + max_bytes: 1073741824 + max_entries: 20000 + prune_interval: 256 compile: enabled: true typecheck: diff --git a/api/registry/dependency_access.go b/api/registry/dependency_access.go new file mode 100644 index 000000000..966902c15 --- /dev/null +++ b/api/registry/dependency_access.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MPL-2.0 + +package registry + +import "context" + +// DependencyAccess controls external dependency access. +type DependencyAccess uint8 + +const ( + // DependencyAccessUnspecified delegates policy selection to the caller. + DependencyAccessUnspecified DependencyAccess = iota + // DependencyAccessOnline permits external resolution and artifact download. + DependencyAccessOnline + // DependencyAccessVerifiedOffline forbids external dependency access. + DependencyAccessVerifiedOffline +) + +type dependencyAccessContextKey struct{} + +// WithDependencyAccess returns a request-scoped dependency access policy. +func WithDependencyAccess(ctx context.Context, access DependencyAccess) context.Context { + return context.WithValue(ctx, dependencyAccessContextKey{}, access) +} + +// DependencyAccessFromContext returns the request-scoped policy. +func DependencyAccessFromContext(ctx context.Context) DependencyAccess { + if ctx == nil { + return DependencyAccessUnspecified + } + access, ok := ctx.Value(dependencyAccessContextKey{}).(DependencyAccess) + if !ok || access > DependencyAccessVerifiedOffline { + return DependencyAccessUnspecified + } + return access +} diff --git a/api/registry/dependency_access_test.go b/api/registry/dependency_access_test.go new file mode 100644 index 000000000..8f6c7c28a --- /dev/null +++ b/api/registry/dependency_access_test.go @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: MPL-2.0 + +package registry + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDependencyAccessContext(t *testing.T) { + require.Equal(t, DependencyAccessUnspecified, DependencyAccessFromContext(context.Background())) + + ctx := WithDependencyAccess(context.Background(), DependencyAccessVerifiedOffline) + require.Equal(t, DependencyAccessVerifiedOffline, DependencyAccessFromContext(ctx)) + require.Equal(t, DependencyAccessUnspecified, DependencyAccessFromContext(nil)) +} diff --git a/boot/components/runtime/lua/engine.go b/boot/components/runtime/lua/engine.go index 47be5705a..de0da78a2 100644 --- a/boot/components/runtime/lua/engine.go +++ b/boot/components/runtime/lua/engine.go @@ -112,13 +112,14 @@ func Engine() boot.Component { func resolveEngineSettings(cfg boot.Config) code.Config { settings := code.Config{ - ProtoCacheSize: 60000, - MainCacheSize: 10000, Cache: cache.Config{ Dir: cache.DefaultDir, Mode: cache.ModeReadWrite, CompileEnabled: true, TypecheckEnabled: true, + MaxBytes: cache.DefaultMaxBytes, + MaxEntries: cache.DefaultMaxEntries, + PruneInterval: cache.DefaultPruneInterval, }, InvalidationWaitTimeout: code.DefaultInvalidationWaitTimeout, } @@ -133,8 +134,6 @@ func resolveEngineSettings(cfg boot.Config) code.Config { ) luaCfg := cfg.Sub("lua") - settings.ProtoCacheSize = luaCfg.GetInt("proto_cache_size", settings.ProtoCacheSize) - settings.MainCacheSize = luaCfg.GetInt("main_cache_size", settings.MainCacheSize) settings.InvalidationWaitTimeout = luaCfg.GetDuration("invalidation_wait_timeout", settings.InvalidationWaitTimeout) typeSystemCfg := luaCfg.Sub("type_system") @@ -153,6 +152,9 @@ func resolveEngineSettings(cfg boot.Config) code.Config { settings.Cache.Mode = cache.ParseMode(luaCfg.GetString("cache.mode", string(settings.Cache.Mode))) settings.Cache.CompileEnabled = luaCfg.GetBool("cache.compile.enabled", settings.Cache.CompileEnabled) settings.Cache.TypecheckEnabled = luaCfg.GetBool("cache.typecheck.enabled", settings.Cache.TypecheckEnabled) + settings.Cache.MaxBytes = int64(luaCfg.GetInt("cache.max_bytes", int(settings.Cache.MaxBytes))) + settings.Cache.MaxEntries = luaCfg.GetInt("cache.max_entries", settings.Cache.MaxEntries) + settings.Cache.PruneInterval = luaCfg.GetInt("cache.prune_interval", settings.Cache.PruneInterval) return settings } diff --git a/boot/components/runtime/lua/engine_test.go b/boot/components/runtime/lua/engine_test.go index 0a253aae7..e1f068b6d 100644 --- a/boot/components/runtime/lua/engine_test.go +++ b/boot/components/runtime/lua/engine_test.go @@ -198,9 +198,6 @@ func TestL04EngineLifecycleIdempotent(t *testing.T) { func TestL05EngineSettingsDefaults(t *testing.T) { settings := resolveEngineSettings(nil) - if settings.ProtoCacheSize != 60000 || settings.MainCacheSize != 10000 { - t.Fatalf("cache sizes = (%d, %d), want (60000, 10000)", settings.ProtoCacheSize, settings.MainCacheSize) - } if settings.TypeCheck.Enabled || settings.TypeCheck.Strict { t.Fatalf("type check defaults = %#v, want disabled and non-strict", settings.TypeCheck) } @@ -210,6 +207,9 @@ func TestL05EngineSettingsDefaults(t *testing.T) { if !settings.Cache.CompileEnabled || !settings.Cache.TypecheckEnabled { t.Fatalf("cache stage defaults = %#v, want both enabled", settings.Cache) } + if settings.Cache.MaxBytes <= 0 || settings.Cache.MaxEntries <= 0 || settings.Cache.PruneInterval <= 0 { + t.Fatalf("cache retention defaults = %#v, want bounded positive limits", settings.Cache) + } if settings.InvalidationWaitTimeout != 30*time.Second { t.Fatalf("invalidation timeout = %v, want 30s", settings.InvalidationWaitTimeout) } diff --git a/boot/deps/hub/dependency_handler.go b/boot/deps/hub/dependency_handler.go index 8e7b0b9be..cf38c91ad 100644 --- a/boot/deps/hub/dependency_handler.go +++ b/boot/deps/hub/dependency_handler.go @@ -1309,6 +1309,9 @@ func (h *DependencyHandler) resolveModules(ctx context.Context, deps []Dependenc provider = h.manifestCache } lockedDigests := h.lockedModuleDigests() + if regapi.DependencyAccessFromContext(ctx) == regapi.DependencyAccessVerifiedOffline { + provider = newLockedManifestProvider(h, lockedVersions, lockedDigests) + } provider = &replacementManifestProvider{ base: provider, handler: h, @@ -1329,6 +1332,10 @@ func (h *DependencyHandler) resolveModules(ctx context.Context, deps []Dependenc if h.logger != nil { h.logger.Error("dependency resolution failed", zap.String("errors", formatResolutionErrors(result.Errors))) } + if regapi.DependencyAccessFromContext(ctx) == regapi.DependencyAccessVerifiedOffline { + module := result.Errors[0].Org + "/" + result.Errors[0].Name + return nil, NewDependencyOfflineError("resolve", strings.Trim(module, "/")) + } return nil, NewDependencyResolutionErrors(result.Errors) } for _, mod := range result.Modules { @@ -1357,6 +1364,17 @@ func (h *DependencyHandler) resolveEffectiveModules( deps []DependencyDefinition, lockedVersions map[string]string, ) ([]ResolvedModule, error) { + if regapi.DependencyAccessFromContext(ctx) == regapi.DependencyAccessVerifiedOffline { + if resolved, ok := h.lockedResolution(deps, lockedVersions); ok { + if h.logger != nil { + h.logger.Debug("using locked dependency resolution", + zap.Int("modules", len(resolved)), + zap.Int("roots", len(deps))) + } + return resolved, nil + } + } + resolved, err := h.resolveModules(ctx, deps, lockedVersions) if err != nil { return nil, err @@ -1667,36 +1685,7 @@ func (p *replacementManifestProvider) localReplacementDependencies(ctx context.C return nil, err } - deps := make([]ManifestDep, 0) - seen := make(map[string]struct{}) - for _, entry := range entries { - if entry.Kind != regapi.NamespaceDependency { - continue - } - def, err := decodeDependency(ctx, transcoder, entry) - if err != nil { - return nil, err - } - if def.Component == "" { - return nil, NewDependencyEntryInvalidError(entry.ID.String(), "component is required", "") - } - name, err := graph.ParseName(def.Component) - if err != nil { - return nil, NewDependencyEntryInvalidError(entry.ID.String(), "invalid component", def.Component) - } - - key := name.String() + "@" + def.Version - if _, ok := seen[key]; ok { - continue - } - seen[key] = struct{}{} - deps = append(deps, ManifestDep{ - Org: name.Organization, - Name: name.Module, - Version: def.Version, - }) - } - return deps, nil + return manifestDependenciesFromEntries(ctx, transcoder, entries) } func loadReplacementEntries( @@ -2147,6 +2136,9 @@ func (h *DependencyHandler) ensureModuleAvailable(ctx context.Context, mod Resol } else if !errors.Is(statErr, os.ErrNotExist) { return "", NewDependencyDownloadError(modKey(mod), statErr) } + if regapi.DependencyAccessFromContext(ctx) == regapi.DependencyAccessVerifiedOffline { + return "", NewDependencyOfflineError("load artifact", modKey(mod)) + } privateDir, err := os.MkdirTemp(h.vendorDir, ".artifact-download-*") if err != nil { @@ -2362,6 +2354,9 @@ func validateDownloadInfo(mod ResolvedModule, info *DownloadInfo) error { // Used both when the resolved manifest carries no URL and to refresh a URL // that expired before the artifact could be downloaded. func (h *DependencyHandler) freshDownloadInfo(ctx context.Context, mod ResolvedModule) (*DownloadInfo, error) { + if regapi.DependencyAccessFromContext(ctx) == regapi.DependencyAccessVerifiedOffline { + return nil, NewDependencyOfflineError("fetch artifact metadata", modKey(mod)) + } downloadURLCtx, cancel := withOptionalTimeout(ctx, h.downloadTimeout) defer cancel() @@ -2850,6 +2845,20 @@ func NewDependencyResolutionError(cause error) apierror.Error { return err } +// NewDependencyOfflineError reports unavailable verified dependency evidence. +func NewDependencyOfflineError(operation, module string) apierror.Error { + details := map[string]any{ + "operation": operation, + "hint": "run an explicit wippy update/install while online, then retry startup", + } + if module != "" { + details["module"] = module + } + return apierror.New(apierror.Invalid, "verified dependency evidence is unavailable during offline startup"). + WithRetryable(apierror.False). + WithDetails(attrs.NewBagFrom(details)) +} + func NewDependencyResolutionErrors(errs []ResolutionError) apierror.Error { details := make([]map[string]any, 0, len(errs)) unauthenticated := false diff --git a/boot/deps/hub/dependency_handler_test.go b/boot/deps/hub/dependency_handler_test.go index 714c13b7b..e9f156ed3 100644 --- a/boot/deps/hub/dependency_handler_test.go +++ b/boot/deps/hub/dependency_handler_test.go @@ -3339,6 +3339,7 @@ replacements: func newTestContext() context.Context { ctx := ctxapi.NewRootContext() + ctx = regapi.WithDependencyAccess(ctx, regapi.DependencyAccessOnline) transcoder := syspayload.NewTranscoder() jsonpayload.Register(transcoder) yamlpayload.Register(transcoder) diff --git a/boot/deps/hub/locked_manifest_provider.go b/boot/deps/hub/locked_manifest_provider.go new file mode 100644 index 000000000..28bf29473 --- /dev/null +++ b/boot/deps/hub/locked_manifest_provider.go @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: MPL-2.0 + +package hub + +import ( + "context" + "fmt" + + "github.com/wippyai/runtime/api/payload" + regapi "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/boot/deps/graph" +) + +// lockedManifestProvider exposes only locally materialized, content-pinned +// modules. It lets the normal resolver validate a graph containing mutable +// replacements without granting that resolver any network capability. +type lockedManifestProvider struct { + handler *DependencyHandler + modules map[string]ResolvedModule +} + +func newLockedManifestProvider( + handler *DependencyHandler, + materializedVersions map[string]string, + lockedDigests map[string]string, +) ManifestProvider { + provider := &lockedManifestProvider{ + handler: handler, + modules: make(map[string]ResolvedModule), + } + if handler == nil || handler.lock == nil { + return provider + } + for _, locked := range handler.lock.GetModules() { + name, err := graph.ParseName(locked.Name) + if err != nil || locked.Version == "" || materializedVersions[locked.Name] != locked.Version { + continue + } + digest := lockedDigests[locked.Name+"@"+locked.Version] + if err := validateModuleArtifactIdentity(name, locked.Version, digest); err != nil || digest == "" { + continue + } + provider.modules[locked.Name] = ResolvedModule{ + Org: name.Organization, + Name: name.Module, + Version: locked.Version, + VersionID: locked.Version, + Source: moduleSourceHub, + Digest: digest, + } + } + return provider +} + +func (p *lockedManifestProvider) GetManifest(ctx context.Context, org, module, constraint string) (*ModuleManifest, error) { + name := org + "/" + module + mod, ok := p.modules[name] + if !ok || !storedVersionSatisfies(mod.Version, constraint) { + return nil, NewDependencyOfflineError("resolve manifest", name) + } + transcoder := payload.GetTranscoder(ctx) + if transcoder == nil { + return nil, ErrDependencyTranscoderMissing + } + entries, err := p.handler.loadEntriesForModule(ctx, transcoder, mod) + if err != nil { + return nil, err + } + deps, err := manifestDependenciesFromEntries(ctx, transcoder, entries) + if err != nil { + return nil, fmt.Errorf("read locked manifest %s@%s: %w", name, mod.Version, err) + } + return &ModuleManifest{ + Org: mod.Org, + Name: mod.Name, + Version: mod.Version, + VersionID: mod.VersionID, + Digest: mod.Digest, + SizeBytes: mod.SizeBytes, + Dependencies: deps, + }, nil +} + +func (p *lockedManifestProvider) ListAllVersions(_ context.Context, org, module string) ([]VersionInfo, error) { + name := org + "/" + module + mod, ok := p.modules[name] + if !ok { + return nil, NewDependencyOfflineError("list versions", name) + } + return []VersionInfo{{Version: mod.Version}}, nil +} + +func manifestDependenciesFromEntries( + ctx context.Context, + transcoder payload.Transcoder, + entries []regapi.Entry, +) ([]ManifestDep, error) { + deps := make([]ManifestDep, 0) + seen := make(map[string]struct{}) + for _, entry := range entries { + if entry.Kind != regapi.NamespaceDependency { + continue + } + def, err := decodeDependency(ctx, transcoder, entry) + if err != nil { + return nil, err + } + if def.Component == "" { + return nil, NewDependencyEntryInvalidError(entry.ID.String(), "component is required", "") + } + name, err := graph.ParseName(def.Component) + if err != nil { + return nil, NewDependencyEntryInvalidError(entry.ID.String(), "invalid component", def.Component) + } + key := name.String() + "@" + def.Version + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + deps = append(deps, ManifestDep{ + Org: name.Organization, Name: name.Module, Version: def.Version, + }) + } + return deps, nil +} diff --git a/boot/deps/hub/locked_resolution.go b/boot/deps/hub/locked_resolution.go new file mode 100644 index 000000000..b8feac5c9 --- /dev/null +++ b/boot/deps/hub/locked_resolution.go @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: MPL-2.0 + +package hub + +import ( + "encoding/hex" + "sort" + "strings" + + "github.com/wippyai/runtime/boot/deps/graph" +) + +// lockedResolution verifies the module selection recorded by wippy update. +func (h *DependencyHandler) lockedResolution( + deps []DependencyDefinition, + materializedVersions map[string]string, +) ([]ResolvedModule, bool) { + if h == nil || h.lock == nil || len(h.replacements) != 0 { + return nil, false + } + + locked := h.lock.GetModules() + if len(locked) == 0 { + return nil, false + } + + selected := make(map[string]ResolvedModule, len(locked)) + for _, mod := range locked { + if mod.Name == "" || mod.Version == "" || mod.Hash == "" { + return nil, false + } + if materializedVersions[mod.Name] != mod.Version { + return nil, false + } + + name, err := graph.ParseName(mod.Name) + if err != nil { + return nil, false + } + algorithm, digest, err := parseExpectedDigest(mod.Hash) + if err != nil || algorithm != "sha256" || len(digest) != 64 { + return nil, false + } + if _, err := hex.DecodeString(digest); err != nil { + return nil, false + } + if _, duplicate := selected[mod.Name]; duplicate { + return nil, false + } + + selected[mod.Name] = ResolvedModule{ + Org: name.Organization, + Name: name.Module, + Version: mod.Version, + VersionID: mod.Version, + Source: moduleSourceHub, + Digest: "sha256:" + strings.ToLower(digest), + } + } + + for _, dep := range deps { + mod, ok := selected[dep.Component] + if !ok || !storedVersionSatisfies(mod.Version, dep.Version) { + return nil, false + } + } + + resolved := make([]ResolvedModule, 0, len(selected)) + for _, mod := range selected { + resolved = append(resolved, mod) + } + sort.Slice(resolved, func(i, j int) bool { + if resolved[i].Org != resolved[j].Org { + return resolved[i].Org < resolved[j].Org + } + return resolved[i].Name < resolved[j].Name + }) + return resolved, true +} diff --git a/boot/deps/hub/locked_resolution_test.go b/boot/deps/hub/locked_resolution_test.go new file mode 100644 index 000000000..bde6b622c --- /dev/null +++ b/boot/deps/hub/locked_resolution_test.go @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: MPL-2.0 + +package hub + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + apierror "github.com/wippyai/runtime/api/error" + regapi "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/boot/deps/lock" + "github.com/wippyai/wapp" + "go.uber.org/zap" +) + +const lockedResolutionDigest = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +func lockedResolutionHandler(t *testing.T, modules []lock.Module) *DependencyHandler { + t.Helper() + lockObj, err := lock.New(filepath.Join(t.TempDir(), "wippy.lock")) + require.NoError(t, err) + lockObj.SetDirectories(lock.Directories{Modules: ".wippy", Src: "src"}) + lockObj.ReplaceModules(modules) + return &DependencyHandler{ + lock: lockObj, + logger: zap.NewNop(), + replacements: make(map[string]lock.Replacement), + } +} + +func TestLockedResolutionSkipsResolverForExactDeployment(t *testing.T) { + handler := lockedResolutionHandler(t, []lock.Module{ + {Name: "acme/app", Version: "1.2.3", Hash: lockedResolutionDigest}, + {Name: "acme/lib", Version: "2.0.0", Hash: "sha256:" + lockedResolutionDigest}, + }) + + resolved, ok := handler.lockedResolution( + []DependencyDefinition{{Component: "acme/app", Version: ">=1.0.0"}}, + map[string]string{"acme/app": "1.2.3", "acme/lib": "2.0.0"}, + ) + require.True(t, ok) + require.Len(t, resolved, 2) + require.Equal(t, "acme/app", resolved[0].Org+"/"+resolved[0].Name) + require.Equal(t, "sha256:"+lockedResolutionDigest, resolved[0].Digest) +} + +func TestResolveEffectiveModulesDoesNotCallHubForExactLock(t *testing.T) { + handler := lockedResolutionHandler(t, []lock.Module{ + {Name: "acme/app", Version: "1.2.3", Hash: lockedResolutionDigest}, + }) + handler.hub = &fakeHub{ + getManifest: func(context.Context, string, string, string) (*ModuleManifest, error) { + t.Fatal("exact locked deployment must not call the Hub resolver") + return nil, nil + }, + } + handler.manifestCache = NewManifestCache(handler.hub) + t.Cleanup(handler.manifestCache.Close) + + ctx := regapi.WithDependencyAccess(context.Background(), regapi.DependencyAccessVerifiedOffline) + resolved, err := handler.resolveEffectiveModules( + ctx, + []DependencyDefinition{{Component: "acme/app", Version: "1.2.3"}}, + map[string]string{"acme/app": "1.2.3"}, + ) + require.NoError(t, err) + require.Len(t, resolved, 1) +} + +func TestLockedResolutionRejectsIncompleteOrDriftedEvidence(t *testing.T) { + tests := []struct { + materialized map[string]string + name string + modules []lock.Module + deps []DependencyDefinition + }{ + { + name: "missing digest", + modules: []lock.Module{{Name: "acme/app", Version: "1.2.3"}}, + deps: []DependencyDefinition{{Component: "acme/app", Version: "1.2.3"}}, + materialized: map[string]string{"acme/app": "1.2.3"}, + }, + { + name: "materialized version drift", + modules: []lock.Module{{Name: "acme/app", Version: "1.2.3", Hash: lockedResolutionDigest}}, + deps: []DependencyDefinition{{Component: "acme/app", Version: "1.2.3"}}, + materialized: map[string]string{"acme/app": "1.2.4"}, + }, + { + name: "root constraint drift", + modules: []lock.Module{{Name: "acme/app", Version: "1.2.3", Hash: lockedResolutionDigest}}, + deps: []DependencyDefinition{{Component: "acme/app", Version: ">=2.0.0"}}, + materialized: map[string]string{"acme/app": "1.2.3"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + handler := lockedResolutionHandler(t, test.modules) + _, ok := handler.lockedResolution(test.deps, test.materialized) + require.False(t, ok) + }) + } +} + +func TestLockedResolutionRejectsMutableReplacements(t *testing.T) { + handler := lockedResolutionHandler(t, []lock.Module{ + {Name: "acme/app", Version: "1.2.3", Hash: lockedResolutionDigest}, + }) + handler.replacements["acme/app"] = lock.Replacement{From: "acme/app", To: "../app"} + + _, ok := handler.lockedResolution( + []DependencyDefinition{{Component: "acme/app", Version: "1.2.3"}}, + map[string]string{"acme/app": "1.2.3"}, + ) + require.False(t, ok) +} + +func TestVerifiedOfflineResolutionNeverFallsBackToHub(t *testing.T) { + handler := lockedResolutionHandler(t, []lock.Module{ + {Name: "acme/app", Version: "1.2.3"}, // Missing digest: not verified. + }) + handler.hub = &fakeHub{ + getManifest: func(context.Context, string, string, string) (*ModuleManifest, error) { + t.Fatal("verified-offline resolution must not call GetManifest") + return nil, nil + }, + listVersions: func(context.Context, string, string) ([]VersionInfo, error) { + t.Fatal("verified-offline resolution must not call ListAllVersions") + return nil, nil + }, + } + + ctx := regapi.WithDependencyAccess(context.Background(), regapi.DependencyAccessVerifiedOffline) + _, err := handler.resolveEffectiveModules( + ctx, + []DependencyDefinition{{Component: "acme/app", Version: "1.2.3"}}, + map[string]string{"acme/app": "1.2.3"}, + ) + require.Error(t, err) + var apiErr apierror.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, apierror.Invalid, apiErr.Kind()) +} + +func TestVerifiedOfflineArtifactMissNeverDownloads(t *testing.T) { + handler := &DependencyHandler{ + hub: &fakeHub{ + getDownload: func(context.Context, *DownloadParams) (*DownloadInfo, error) { + t.Fatal("verified-offline artifact load must not call GetDownloadURL") + return nil, nil + }, + downloadFile: func(context.Context, string, string) error { + t.Fatal("verified-offline artifact load must not download") + return nil + }, + }, + logger: zap.NewNop(), + vendorDir: t.TempDir(), + replacements: make(map[string]lock.Replacement), + } + ctx := regapi.WithDependencyAccess(context.Background(), regapi.DependencyAccessVerifiedOffline) + _, err := handler.ensureModuleAvailable(ctx, ResolvedModule{ + Org: "acme", Name: "app", Version: "1.2.3", + Digest: "sha256:" + lockedResolutionDigest, + }) + require.Error(t, err) + var apiErr apierror.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, apierror.Invalid, apiErr.Kind()) +} + +func TestVerifiedOfflineResolverUsesInstalledModuleGraph(t *testing.T) { + vendorDir := t.TempDir() + artifacts := map[string][]byte{ + "app": buildWappBytes(t, []wapp.Entry{{ + ID: wapp.NewID("acme.app", "lib"), Kind: regapi.NamespaceDependency, + Data: map[string]any{"component": "acme/lib", "version": "v1.0.0"}, + }}), + "lib": buildWappBytes(t, []wapp.Entry{{ID: wapp.NewID("acme.lib", "service"), Kind: "service"}}), + } + modules := make([]lock.Module, 0, len(artifacts)) + for _, name := range []string{"app", "lib"} { + sum := sha256.Sum256(artifacts[name]) + digest := "sha256:" + hex.EncodeToString(sum[:]) + modules = append(modules, lock.Module{Name: "acme/" + name, Version: "v1.0.0", Hash: digest}) + path := filepath.Join(vendorDir, "acme", name+"-v1.0.0.wapp") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, artifacts[name], 0o600)) + } + handler := lockedResolutionHandler(t, modules) + handler.vendorDir = vendorDir + handler.replacements["unused/local"] = lock.Replacement{From: "unused/local", To: t.TempDir()} + handler.hub = &fakeHub{ + getManifest: func(context.Context, string, string, string) (*ModuleManifest, error) { + t.Fatal("installed offline resolution must not query Hub manifests") + return nil, nil + }, + listVersions: func(context.Context, string, string) ([]VersionInfo, error) { + t.Fatal("installed offline resolution must not query Hub versions") + return nil, nil + }, + } + + ctx := regapi.WithDependencyAccess(newTestContext(), regapi.DependencyAccessVerifiedOffline) + resolved, err := handler.resolveEffectiveModules(ctx, + []DependencyDefinition{{Component: "acme/app", Version: "v1.0.0"}}, + map[string]string{"acme/app": "v1.0.0", "acme/lib": "v1.0.0"}, + ) + require.NoError(t, err) + require.Len(t, resolved, 2) +} diff --git a/boot/deps/hub/replacement_restart_e2e_test.go b/boot/deps/hub/replacement_restart_e2e_test.go index 242aba7d7..3b11e49c0 100644 --- a/boot/deps/hub/replacement_restart_e2e_test.go +++ b/boot/deps/hub/replacement_restart_e2e_test.go @@ -12,8 +12,10 @@ import ( "github.com/stretchr/testify/require" "github.com/wippyai/runtime/api/payload" regapi "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/internal/version" registryimpl "github.com/wippyai/runtime/system/registry" regexp "github.com/wippyai/runtime/system/registry/expansion" + "github.com/wippyai/runtime/system/registry/history/memory" historysqlite "github.com/wippyai/runtime/system/registry/history/sqlite" "github.com/wippyai/runtime/system/registry/topology" "go.uber.org/zap" @@ -88,6 +90,13 @@ replacements: Kind: regapi.NamespaceDependency, Data: payload.New(map[string]any{"component": "local/mod", "version": "v0.1.0"}), } + fresh := newRegistry(memory.New(), newHandler()) + startupCtx := regapi.WithDependencyAccess(newTestContext(), regapi.DependencyAccessUnspecified) + require.NoError(t, fresh.LoadState(startupCtx, regapi.State{root}, version.New(0))) + _, err = fresh.GetEntry(regapi.NewID("local.mod", "svc")) + require.NoError(t, err) + require.Zero(t, hubCalls) + version, err := registry.Apply(ctx, regapi.ChangeSet{{Kind: regapi.EntryCreate, Entry: root}}) require.NoError(t, err) require.Zero(t, hubCalls) @@ -107,7 +116,8 @@ replacements: require.NoError(t, err) t.Cleanup(func() { _ = history.Close() }) restarted := newRegistry(history, newHandler()) - require.NoError(t, restarted.LoadState(ctx, nil, version)) + restartCtx := regapi.WithDependencyAccess(newTestContext(), regapi.DependencyAccessVerifiedOffline) + require.NoError(t, restarted.LoadState(restartCtx, nil, version)) require.Zero(t, hubCalls) reloadedEntry, err := restarted.GetEntry(entryID) diff --git a/cmd/wippy/cmd/lint.go b/cmd/wippy/cmd/lint.go index 4669fc809..4e3faa014 100644 --- a/cmd/wippy/cmd/lint.go +++ b/cmd/wippy/cmd/lint.go @@ -31,6 +31,7 @@ import ( appinit "github.com/wippyai/runtime/cmd/internal/app" clilogger "github.com/wippyai/runtime/cmd/internal/logger" "github.com/wippyai/runtime/runtime/lua/code" + "github.com/wippyai/runtime/runtime/lua/code/cache" "github.com/wippyai/runtime/runtime/lua/code/lint" _ "github.com/wippyai/runtime/runtime/lua/code/lint/rules" // register lint rules "github.com/wippyai/runtime/runtime/lua/component" @@ -208,6 +209,22 @@ type lintConfig struct { workers int } +const maxLintWorkers = 8 + +func boundedLintWorkers(procs int) int { + if procs < 1 { + return 1 + } + if procs > maxLintWorkers { + return maxLintWorkers + } + return procs +} + +func defaultLintWorkers() int { + return boundedLintWorkers(runtime.GOMAXPROCS(0)) +} + // luaEntryKinds are the entry kinds that contain Lua code. var luaEntryKinds = []string{ "function.lua", @@ -228,7 +245,12 @@ func runLint(cmd *cobra.Command, _ []string) error { return err } - ctx, loader, err := bootstrapLintContext() + runtimeCfg, err := loadRuntimeConfig(cmd, zap.NewNop()) + if err != nil { + return err + } + + ctx, loader, err := bootstrapLintContext(runtimeCfg) if err != nil { return err } @@ -240,7 +262,7 @@ func runLint(cmd *cobra.Command, _ []string) error { defer func() { _ = loader.Shutdown(ctx) }() } - luaEntries, reportSet, err := loadLuaEntries(cmd, opts.lockFile, opts.nsFilters) + luaEntries, reportSet, err := loadLuaEntries(cmd, runtimeCfg, opts.lockFile, opts.nsFilters) if err != nil { return err } @@ -253,7 +275,7 @@ func runLint(cmd *cobra.Command, _ []string) error { } cfg := lintConfig{ minSeverity: opts.minSeverity, - workers: runtime.NumCPU(), + workers: defaultLintWorkers(), } var result *LintResult @@ -267,6 +289,11 @@ func runLint(cmd *cobra.Command, _ []string) error { } result = applyFilters(result, opts.codeFilters, opts.limit) + if pruner, ok := lcache.store.(cache.Pruner); ok && lintCacheAllowsWrite(lcache) { + if err := pruner.Prune(); err != nil { + return err + } + } return outputResults(result, opts) } @@ -310,7 +337,7 @@ func parseLintFlags(cmd *cobra.Command) (lintOptions, error) { }, nil } -func bootstrapLintContext() (ctx context.Context, loader *bootpkg.Loader, err error) { +func bootstrapLintContext(cfg boot.Config) (ctx context.Context, loader *bootpkg.Loader, err error) { logger, err := clilogger.CreateLogger(clilogger.Config{ Silent: true, AppStartTime: appStartTime, @@ -319,14 +346,6 @@ func bootstrapLintContext() (ctx context.Context, loader *bootpkg.Loader, err er return nil, nil, NewCreateLoggerError(err) } - cfg, err := loadBootConfig() - if err != nil { - return nil, nil, err - } - if cfg == nil { - cfg = createDefaultConfig() - } - bctx, err := bootpkg.NewBootstrapContext(logger, cfg) if err != nil { return nil, nil, NewInitializeBootstrapContextError(err) @@ -364,17 +383,13 @@ func bootstrapLintContext() (ctx context.Context, loader *bootpkg.Loader, err er return bctx, loader, nil } -func loadLuaEntries(cmd *cobra.Command, lockFile string, nsFilters []string) ([]regapi.Entry, map[regapi.ID]bool, error) { +func loadLuaEntries(cmd *cobra.Command, runtimeCfg boot.Config, lockFile string, nsFilters []string) ([]regapi.Entry, map[regapi.ID]bool, error) { logger := zap.NewNop() app, err := appinit.Init(cmd.Context(), verbose, veryVerbose, console, silentLogs, appStartTime) if err != nil { return nil, nil, NewInitAppError(err) } - runtimeCfg, err := loadRuntimeConfig(cmd, logger) - if err != nil { - return nil, nil, err - } boot.WithConfig(app.Ctx, runtimeCfg) lockPath, lockObj, err := loadValidatedLock(".", lockFile, runtimeCfg, logger) @@ -405,6 +420,11 @@ func createLinter(ctx context.Context, enableRules bool) (*lint.Linter, lintCach Enabled: true, Strict: true, } + if cm != nil { + if runtimeTypeCfg := cm.TypeCheckConfig(); runtimeTypeCfg.Enabled { + typeCfg = runtimeTypeCfg + } + } typeChecker := code.NewTypeChecker(typeCfg, mods) var registry *lint.Registry @@ -420,19 +440,9 @@ func createLinter(ctx context.Context, enableRules bool) (*lint.Linter, lintCach lcache.cfg = cm.CacheConfig() } lcache.typecheckHash = code.TypecheckConfigHash(typeCfg) - lcache.builtinModules = make([]string, 0, len(mods)) - builtinManifests := make(map[string]*io.Manifest) - for _, mod := range mods { - if mod == nil || mod.Types == nil { - continue - } - manifest := mod.Types() - if manifest == nil { - continue - } - lcache.builtinModules = append(lcache.builtinModules, mod.Name) - builtinManifests[mod.Name] = manifest - } + var builtinManifests map[string]*io.Manifest + lcache.builtinModules, builtinManifests = lintBuiltinInventory(mods) + lcache.builtinHash = code.BuiltinManifestHash(builtinManifests) // requireBuiltins is the set of modules a scoped require resolves without an // explicit import/module declaration. It mirrors the runtime ambient base @@ -446,11 +456,30 @@ func createLinter(ctx context.Context, enableRules bool) (*lint.Linter, lintCach for _, name := range component.ExecutableAmbientModuleNames() { lcache.requireBuiltins[name] = struct{}{} } - lcache.builtinHash = code.BuiltinManifestHash(builtinManifests) return lint.New(typeChecker, registry), lcache } +func lintBuiltinInventory(mods []*luaapi.ModuleDef) ([]string, map[string]*io.Manifest) { + names := make([]string, 0, len(mods)) + manifests := make(map[string]*io.Manifest) + for _, mod := range mods { + if mod == nil || mod.Name == "" { + continue + } + names = append(names, mod.Name) + if mod.Types == nil { + continue + } + manifest := mod.Types() + if manifest == nil { + continue + } + manifests[mod.Name] = manifest + } + return names, manifests +} + func applyFilters(result *LintResult, codeFilters []string, limit int) *LintResult { if len(codeFilters) > 0 { result = filterByCode(result, codeFilters) @@ -533,6 +562,12 @@ func runLintSimple(luaEntries []regapi.Entry, reportSet map[regapi.ID]bool, lint // lintEntries is the core linting loop. If prog is non-nil, sends UI updates. func lintEntries(luaEntries []regapi.Entry, reportSet map[regapi.ID]bool, linter *lint.Linter, lcache lintCache, cfg lintConfig, prog *tea.Program) *LintResult { result := &LintResult{TotalEntries: len(luaEntries)} + workers := cfg.workers + if workers < 1 { + workers = defaultLintWorkers() + } else { + workers = boundedLintWorkers(workers) + } levels, _ := topology.LevelSortEntriesByDependency(luaEntries, &luaImportResolver{}) entryDataMap := make(map[regapi.ID]entryData, len(luaEntries)) @@ -593,7 +628,7 @@ func lintEntries(luaEntries []regapi.Entry, reportSet map[regapi.ID]bool, linter } results := make([]entryResult, len(levelEntries)) - sem := make(chan struct{}, cfg.workers) + sem := make(chan struct{}, workers) var wg sync.WaitGroup for i, entry := range levelEntries { @@ -672,29 +707,32 @@ func lintOneEntry(entry regapi.Entry, data entryData, linter *lint.Linter, manif var cachedManifest *io.Manifest var cachedDiagnostics []diag.Diagnostic + typecheckCacheHit := false if tcFP := fps.typecheck[entry.ID]; tcFP != "" { if manifest, diags, ok := lintLoadTypecheckCache(lcache, entry.ID, tcFP); ok { cachedManifest = manifest cachedDiagnostics = diags + typecheckCacheHit = true } } enableTypecheck := cachedDiagnostics == nil lintResult := linter.CheckParsedWithTypecheck(stmts, entryID, imports, enableTypecheck) linter.ClearCache() - - requireDiags := lintRequireDeclarations(stmts, entryID, data, lcache.requireBuiltins) - if len(requireDiags) > 0 { - lintResult.Diagnostics = append(requireDiags, lintResult.Diagnostics...) - } + typeDiags := filterTypecheckDiagnostics(lintResult.Diagnostics) if cachedDiagnostics != nil { lintResult.Manifest = cachedManifest lintResult.Diagnostics = append(cachedDiagnostics, lintResult.Diagnostics...) + typeDiags = cachedDiagnostics } - typeDiags := filterTypecheckDiagnostics(lintResult.Diagnostics) - if lintResult.Manifest != nil { + requireDiags := lintRequireDeclarations(stmts, entryID, data, lcache.requireBuiltins) + if len(requireDiags) > 0 { + lintResult.Diagnostics = append(requireDiags, lintResult.Diagnostics...) + } + + if lintResult.Manifest != nil && !typecheckCacheHit { lintSaveTypecheckCache(lcache, entry, data, fps.typecheck[entry.ID], fps.typeDeps[entry.ID], lintResult.Manifest, typeDiags) } @@ -984,10 +1022,14 @@ func extractEntryData(entry regapi.Entry) entryData { // Fast path: loader entries usually carry golang map payloads. if m, ok := entry.Data.Data().(map[string]any); ok { - return entryDataFromMap(m) + data := entryDataFromMap(m) + data.Method = code.EffectiveMethod(entry.Kind, data.Method) + return data } if m, ok := entry.Data.Data().(map[string]interface{}); ok { - return entryDataFromMap(m) + data := entryDataFromMap(m) + data.Method = code.EffectiveMethod(entry.Kind, data.Method) + return data } var cfg struct { @@ -1009,7 +1051,7 @@ func extractEntryData(entry regapi.Entry) entryData { imports[mod] = regapi.NewID("", mod) } - return entryData{Source: cfg.Source, Imports: imports, Method: cfg.Method} + return entryData{Source: cfg.Source, Imports: imports, Method: code.EffectiveMethod(entry.Kind, cfg.Method)} } func entryDataFromMap(m map[string]any) entryData { diff --git a/cmd/wippy/cmd/lint_cache.go b/cmd/wippy/cmd/lint_cache.go index 09fdfea0a..79a06d631 100644 --- a/cmd/wippy/cmd/lint_cache.go +++ b/cmd/wippy/cmd/lint_cache.go @@ -20,10 +20,10 @@ import ( type lintCache struct { store cache.Store requireBuiltins map[string]struct{} + builtinModules []string builtinHash string typecheckHash string cfg cache.Config - builtinModules []string } type lintFingerprints struct { diff --git a/cmd/wippy/cmd/lint_scale_test.go b/cmd/wippy/cmd/lint_scale_test.go new file mode 100644 index 000000000..3534c719c --- /dev/null +++ b/cmd/wippy/cmd/lint_scale_test.go @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cmd + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/event" + regapi "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/runtime/lua/code" + "github.com/wippyai/runtime/runtime/lua/code/cache" + "github.com/wippyai/runtime/runtime/lua/code/lint" + "go.uber.org/zap" +) + +func TestLintWarmCacheIsReadOnly(t *testing.T) { + runLintCacheHarness(t, 32, false) +} + +func TestLintWarmCacheDoesNotDuplicateRequireDiagnostics(t *testing.T) { + dir := t.TempDir() + typeCfg := code.TypeCheckConfig{Enabled: true, Strict: true} + lcache := lintCache{ + store: cache.NewDiskStore(dir), + cfg: cache.Config{ + Enabled: true, CompileEnabled: true, TypecheckEnabled: true, + }, + typecheckHash: code.TypecheckConfigHash(typeCfg), + builtinHash: code.BuiltinManifestHash(nil), + requireBuiltins: map[string]struct{}{}, + } + entry := makeLuaSourceEntry( + regapi.NewID("cache", "require"), nil, `return require("undeclared")`, + ) + entries := []regapi.Entry{entry} + report := map[regapi.ID]bool{entry.ID: true} + linter := lint.New(code.NewTypeChecker(typeCfg, nil), lint.NewRegistry()) + + cold := lintEntries(entries, report, linter, lcache, lintConfig{minSeverity: severityError}, nil) + warm := lintEntries(entries, report, linter, lcache, lintConfig{minSeverity: severityError}, nil) + + require.Equal(t, cold.Diagnostics, warm.Diagnostics) + require.Len(t, warm.Diagnostics, 1) + require.Equal(t, "E0007", warm.Diagnostics[0].Code) +} + +func TestLintLargeApplicationCacheHarness(t *testing.T) { + raw := os.Getenv("WIPPY_LINT_SCALE_ENTRIES") + if raw == "" { + t.Skip("set WIPPY_LINT_SCALE_ENTRIES to run the large-application harness") + } + count, err := strconv.Atoi(raw) + require.NoError(t, err) + require.Positive(t, count) + runLintCacheHarness(t, count, os.Getenv("WIPPY_LINT_SCALE_SHAPE") == "chain") +} + +func runLintCacheHarness(t *testing.T, count int, chain bool) { + t.Helper() + dir := t.TempDir() + store := cache.NewBoundedDiskStore(dir, 1<<30, count*3, 64) + typeCfg := code.TypeCheckConfig{Enabled: true, Strict: true} + lcache := lintCache{ + store: store, + cfg: cache.Config{Enabled: true, CompileEnabled: true, TypecheckEnabled: true}, + typecheckHash: code.TypecheckConfigHash(typeCfg), + builtinHash: code.BuiltinManifestHash(nil), + requireBuiltins: map[string]struct{}{}, + } + entries := make([]regapi.Entry, 0, count) + report := make(map[regapi.ID]bool, count) + ids := make([]regapi.ID, count) + for i := 0; i < count; i++ { + ids[i] = regapi.NewID("scale", fmt.Sprintf("entry_%06d", i)) + } + for i, id := range ids { + var imports map[string]regapi.ID + if chain && i > 0 { + imports = map[string]regapi.ID{"previous": ids[i-1]} + } else if !chain && i == len(ids)-1 && len(ids) > 1 { + imports = make(map[string]regapi.ID, len(ids)-1) + for depIndex, depID := range ids[:len(ids)-1] { + imports[fmt.Sprintf("dep_%06d", depIndex)] = depID + } + } + entries = append(entries, makeLuaEntry(id, imports)) + report[id] = true + } + + linter := lint.New(code.NewTypeChecker(typeCfg, nil), lint.NewRegistry()) + started := time.Now() + cold := lintEntries(entries, report, linter, lcache, lintConfig{minSeverity: severityError}, nil) + require.Zero(t, cold.ErrorCount) + require.NoError(t, store.Prune()) + coldDuration := time.Since(started) + before := cacheFileModTimes(t, dir) + + started = time.Now() + warm := lintEntries(entries, report, linter, lcache, lintConfig{minSeverity: severityError}, nil) + require.Zero(t, warm.ErrorCount) + warmDuration := time.Since(started) + after := cacheFileModTimes(t, dir) + require.Equal(t, before, after, "a warm lint must not rewrite cache hits") + + manager, err := code.NewCodeManager(zap.NewNop(), lintCacheEventBus{}, code.Config{ + Cache: cache.Config{ + Dir: dir, Enabled: true, CompileEnabled: true, TypecheckEnabled: true, + MaxBytes: 1 << 30, MaxEntries: count * 3, PruneInterval: 64, + }, + TypeCheck: typeCfg, + }) + require.NoError(t, err) + for _, entry := range entries { + data := extractEntryData(entry) + deps := make([]code.Import, 0, len(data.Imports)) + for alias, id := range data.Imports { + deps = append(deps, code.Import{Alias: alias, ID: id}) + } + require.NoError(t, manager.AddNode(context.Background(), code.Node{ + ID: entry.ID, Kind: entry.Kind, Source: data.Source, Method: data.Method, + }, deps)) + } + runtimeStarted := time.Now() + _, err = manager.Compile(entries[len(entries)-1].ID, nil) + require.NoError(t, err) + runtimeDuration := time.Since(runtimeStarted) + runtimeAfter := cacheFileModTimes(t, dir) + require.Equal(t, after, runtimeAfter, "runtime must consume lint artifacts without rewriting them") + + shape := "fan-in" + if chain { + shape = "chain" + } + t.Logf("entries=%d shape=%s cold=%s warm=%s runtime=%s files=%d", + count, shape, coldDuration, warmDuration, runtimeDuration, len(runtimeAfter)) +} + +type lintCacheEventBus struct{} + +func (lintCacheEventBus) Send(context.Context, event.Event) {} + +func (lintCacheEventBus) Subscribe(context.Context, event.System, chan<- event.Event) (event.SubscriberID, error) { + return "lint-cache-test", nil +} + +func (lintCacheEventBus) SubscribeP(context.Context, event.System, event.Kind, chan<- event.Event) (event.SubscriberID, error) { + return "lint-cache-test", nil +} + +func (lintCacheEventBus) Unsubscribe(context.Context, event.SubscriberID) {} + +func cacheFileModTimes(t *testing.T, root string) map[string]time.Time { + t.Helper() + out := make(map[string]time.Time) + require.NoError(t, filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if !entry.Type().IsRegular() { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + out[rel] = info.ModTime() + return nil + })) + return out +} diff --git a/cmd/wippy/cmd/lint_test.go b/cmd/wippy/cmd/lint_test.go index f8c244354..e8764065a 100644 --- a/cmd/wippy/cmd/lint_test.go +++ b/cmd/wippy/cmd/lint_test.go @@ -8,8 +8,10 @@ import ( "testing" "github.com/wippyai/go-lua/compiler/parse" + "github.com/wippyai/go-lua/types/io" "github.com/wippyai/runtime/api/payload" "github.com/wippyai/runtime/api/registry" + luaapi "github.com/wippyai/runtime/api/runtime/lua" "github.com/wippyai/runtime/runtime/lua/component" "github.com/wippyai/runtime/runtime/lua/engine" transcoder "github.com/wippyai/runtime/system/payload" @@ -23,11 +25,15 @@ func ambientRequireBuiltins() []string { } func makeLuaEntry(id registry.ID, imports map[string]registry.ID) registry.Entry { + return makeLuaSourceEntry(id, imports, "return {}") +} + +func makeLuaSourceEntry(id registry.ID, imports map[string]registry.ID, source string) registry.Entry { cfg := struct { Imports map[string]registry.ID `json:"imports,omitempty"` Source string `json:"source"` }{ - Source: "return {}", + Source: source, Imports: imports, } payloadjson.Register(transcoder.GlobalTranscoder()) @@ -42,6 +48,37 @@ func makeLuaEntry(id registry.ID, imports map[string]registry.ID) registry.Entry } } +func TestLintBuiltinInventoryIncludesUntypedModules(t *testing.T) { + typed := &luaapi.ModuleDef{Name: "typed", Types: func() *io.Manifest { return io.NewManifest("typed") }} + untyped := &luaapi.ModuleDef{Name: "untyped"} + names, manifests := lintBuiltinInventory([]*luaapi.ModuleDef{typed, untyped}) + if len(names) != 2 || names[0] != "typed" || names[1] != "untyped" { + t.Fatalf("unexpected builtin inventory: %v", names) + } + if manifests["typed"] == nil { + t.Fatal("typed module manifest missing") + } + if _, ok := manifests["untyped"]; ok { + t.Fatal("untyped module must not fabricate a manifest") + } +} + +func TestExtractEntryDataUsesRuntimeLibraryMethod(t *testing.T) { + payloadjson.Register(transcoder.GlobalTranscoder()) + raw, err := json.Marshal(map[string]any{"source": "return {}", "method": "ignored"}) + if err != nil { + t.Fatal(err) + } + entry := registry.Entry{ + ID: registry.NewID("app", "library"), + Kind: luaapi.Library, + Data: payload.NewPayload(raw, payload.JSON), + } + if method := extractEntryData(entry).Method; method != "" { + t.Fatalf("library method = %q, want empty runtime method", method) + } +} + func TestExpandLuaEntriesByImports_IncludesDeps(t *testing.T) { depID := registry.NewID("ns.dep", "dep") rootID := registry.NewID("ns.root", "root") diff --git a/cmd/wippy/cmd/lint_workers_test.go b/cmd/wippy/cmd/lint_workers_test.go new file mode 100644 index 000000000..c4a405f63 --- /dev/null +++ b/cmd/wippy/cmd/lint_workers_test.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cmd + +import "testing" + +func TestBoundedLintWorkers(t *testing.T) { + for _, test := range []struct { + name string + procs int + want int + }{ + {name: "zero", procs: 0, want: 1}, + {name: "negative", procs: -1, want: 1}, + {name: "available", procs: 4, want: 4}, + {name: "cap", procs: maxLintWorkers + 1, want: maxLintWorkers}, + } { + t.Run(test.name, func(t *testing.T) { + if got := boundedLintWorkers(test.procs); got != test.want { + t.Fatalf("boundedLintWorkers(%d) = %d, want %d", test.procs, got, test.want) + } + }) + } +} + +func TestDefaultLintWorkersIsBounded(t *testing.T) { + got := defaultLintWorkers() + if got < 1 || got > maxLintWorkers { + t.Fatalf("defaultLintWorkers() = %d, want [1,%d]", got, maxLintWorkers) + } +} diff --git a/cmd/wippy/cmd/run_test.go b/cmd/wippy/cmd/run_test.go index 749a909e8..5f8b09b11 100644 --- a/cmd/wippy/cmd/run_test.go +++ b/cmd/wippy/cmd/run_test.go @@ -16,7 +16,7 @@ import ( func TestLoadBootConfigSetsConfigDir(t *testing.T) { tempDir := t.TempDir() cfgPath := filepath.Join(tempDir, "wippy.yaml") - cfgBody := []byte("version: \"1.0\"\nlua:\n proto_cache_size: 1\n") + cfgBody := []byte("version: \"1.0\"\nlua:\n cache:\n max_entries: 1\n") require.NoError(t, os.WriteFile(cfgPath, cfgBody, 0o644)) prevProfiler := profiler diff --git a/cmd/wippy/main.go b/cmd/wippy/main.go index b75b780c3..ee49005e0 100644 --- a/cmd/wippy/main.go +++ b/cmd/wippy/main.go @@ -5,7 +5,6 @@ package main import ( "fmt" "os" - "runtime" sqlitevec "github.com/asg017/sqlite-vec-go-bindings/cgo" "github.com/wippyai/runtime/cmd/wippy/cmd" @@ -18,8 +17,6 @@ import ( func main() { sqlitevec.Auto() - runtime.GOMAXPROCS(runtime.NumCPU()) - if err := cmd.Execute(); err != nil { errStr := err.Error() if cmd.IsConsoleMode() { diff --git a/go.mod b/go.mod index 9b900ccdf..6941cd849 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,7 @@ require ( github.com/tree-sitter/tree-sitter-php v0.24.2 github.com/tree-sitter/tree-sitter-python v0.25.0 github.com/tree-sitter/tree-sitter-typescript v0.23.2 - github.com/wippyai/go-lua v1.5.16 + github.com/wippyai/go-lua v1.5.17 github.com/wippyai/module-registry-proto-go v0.0.1 github.com/wippyai/tree-sitter-markdown v0.0.3 github.com/wippyai/tree-sitter-sql v0.0.4 diff --git a/go.sum b/go.sum index 52a4520bc..48af311c1 100644 --- a/go.sum +++ b/go.sum @@ -550,8 +550,8 @@ github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8 github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA= github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= -github.com/wippyai/go-lua v1.5.16 h1:FOjZfd3H73MLQY0/nnKKW4Fik/rZDgCcg0L3W0Yr6Iw= -github.com/wippyai/go-lua v1.5.16/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.5.17 h1:Nd/ym4A8dcmVWcOnu7x8qi22MlPmNbHgE6+Rb8As4DQ= +github.com/wippyai/go-lua v1.5.17/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/module-registry-proto-go v0.0.1 h1:EYnW8MTrI/gs7TJ0ilTVgJal3e0NWmXQigeamrXu0mk= github.com/wippyai/module-registry-proto-go v0.0.1/go.mod h1:p4ihYQKRQuqRVLxaH1YL0IEnkz4zKdyfVADOwvA5Tno= github.com/wippyai/tree-sitter-markdown v0.0.3 h1:u6e53+hzPSS848Xhm+I48SwtvWQHrC21wDrogdwCwuc= diff --git a/runtime/lua/code/cache/config.go b/runtime/lua/code/cache/config.go index 3bcae9526..8733b0f5c 100644 --- a/runtime/lua/code/cache/config.go +++ b/runtime/lua/code/cache/config.go @@ -5,6 +5,12 @@ package cache // DefaultDir is the default on-disk cache directory relative to the working dir. const DefaultDir = ".wippy/cache/lua" +const ( + DefaultMaxBytes int64 = 1 << 30 + DefaultMaxEntries = 20_000 + DefaultPruneInterval = 256 +) + // Mode controls cache read/write behavior. type Mode string @@ -21,6 +27,9 @@ type Config struct { Enabled bool CompileEnabled bool TypecheckEnabled bool + MaxBytes int64 + MaxEntries int + PruneInterval int } // Normalize applies default values. @@ -34,6 +43,15 @@ func (c Config) Normalize() Config { if c.Mode == "" { c.Mode = ModeReadWrite } + if c.MaxBytes <= 0 { + c.MaxBytes = DefaultMaxBytes + } + if c.MaxEntries <= 0 { + c.MaxEntries = DefaultMaxEntries + } + if c.PruneInterval <= 0 { + c.PruneInterval = DefaultPruneInterval + } return c } diff --git a/runtime/lua/code/cache/disk_store.go b/runtime/lua/code/cache/disk_store.go index 4dd81ed4a..37166a5b2 100644 --- a/runtime/lua/code/cache/disk_store.go +++ b/runtime/lua/code/cache/disk_store.go @@ -7,6 +7,9 @@ import ( "errors" "os" "path/filepath" + "sort" + "sync" + "sync/atomic" "time" ) @@ -19,21 +22,47 @@ const ( // DiskStore stores cache entries on disk. type DiskStore struct { - root string + root string + maxBytes int64 + maxEntries int + pruneInterval uint64 + writes atomic.Uint64 + mu sync.RWMutex } // NewDiskStore creates a disk-backed cache store. func NewDiskStore(dir string) *DiskStore { - return &DiskStore{root: dir} + return NewBoundedDiskStore(dir, DefaultMaxBytes, DefaultMaxEntries, DefaultPruneInterval) +} + +// NewBoundedDiskStore creates a disk cache with bounded retained generations. +func NewBoundedDiskStore(dir string, maxBytes int64, maxEntries, pruneInterval int) *DiskStore { + if maxBytes <= 0 { + maxBytes = DefaultMaxBytes + } + if maxEntries <= 0 { + maxEntries = DefaultMaxEntries + } + if pruneInterval <= 0 { + pruneInterval = DefaultPruneInterval + } + return &DiskStore{ + root: dir, maxBytes: maxBytes, maxEntries: maxEntries, + pruneInterval: uint64(pruneInterval), + } } // Delete removes a cache entry by key. func (s *DiskStore) Delete(key string) error { + s.mu.Lock() + defer s.mu.Unlock() return os.RemoveAll(s.entryDir(key)) } // Get retrieves a cache entry by key. func (s *DiskStore) Get(key string) (*Entry, bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() entryDir := s.entryDir(key) metaPath := filepath.Join(entryDir, metaFile) metaData, err := os.ReadFile(metaPath) @@ -70,6 +99,19 @@ func (s *DiskStore) Put(key string, entry *Entry) error { if entry == nil { return nil } + s.mu.RLock() + err := s.put(key, entry) + s.mu.RUnlock() + if err != nil { + return err + } + if s.writes.Add(1)%s.pruneInterval == 0 { + return s.Prune() + } + return nil +} + +func (s *DiskStore) put(key string, entry *Entry) error { entryDir := s.entryDir(key) if err := os.MkdirAll(entryDir, 0o755); err != nil { return err @@ -110,6 +152,79 @@ func (s *DiskStore) Put(key string, entry *Entry) error { return writeFileAtomic(entryDir, metaFile, metaData) } +type diskEntryInfo struct { + created time.Time + path string + size int64 +} + +// Prune removes the oldest retained generations until both configured limits +// are satisfied. Cache eviction never affects correctness: a miss recompiles. +func (s *DiskStore) Prune() error { + s.mu.Lock() + defer s.mu.Unlock() + + root := filepath.Join(s.root, "v1", "entries") + dirs, err := os.ReadDir(root) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + entries := make([]diskEntryInfo, 0, len(dirs)) + var total int64 + for _, dir := range dirs { + if !dir.IsDir() { + continue + } + path := filepath.Join(root, dir.Name()) + info := diskEntryInfo{path: path} + walkErr := filepath.WalkDir(path, func(_ string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.Type().IsRegular() { + stat, statErr := d.Info() + if statErr != nil { + return statErr + } + info.size += stat.Size() + if info.created.IsZero() || stat.ModTime().Before(info.created) { + info.created = stat.ModTime() + } + } + return nil + }) + if walkErr != nil { + return walkErr + } + if data, readErr := os.ReadFile(filepath.Join(path, metaFile)); readErr == nil { + var meta Meta + if json.Unmarshal(data, &meta) == nil && !meta.CreatedAt.IsZero() { + info.created = meta.CreatedAt + } + } + total += info.size + entries = append(entries, info) + } + sort.Slice(entries, func(i, j int) bool { + if entries[i].created.Equal(entries[j].created) { + return entries[i].path < entries[j].path + } + return entries[i].created.Before(entries[j].created) + }) + for len(entries) > s.maxEntries || total > s.maxBytes { + oldest := entries[0] + entries = entries[1:] + if err := os.RemoveAll(oldest.path); err != nil { + return err + } + total -= oldest.size + } + return nil +} + func (s *DiskStore) entryDir(key string) string { return filepath.Join(s.root, "v1", "entries", key) } diff --git a/runtime/lua/code/cache/disk_store_test.go b/runtime/lua/code/cache/disk_store_test.go index 3f165c335..40363f8f3 100644 --- a/runtime/lua/code/cache/disk_store_test.go +++ b/runtime/lua/code/cache/disk_store_test.go @@ -90,3 +90,42 @@ func TestDiskStoreDelete(t *testing.T) { require.NoError(t, err) assert.False(t, ok) } + +func TestDiskStorePrunesOldestGenerationByEntryLimit(t *testing.T) { + store := NewBoundedDiskStore(t.TempDir(), 1<<20, 2, 1) + base := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC) + for i, key := range []string{"old", "middle", "new"} { + require.NoError(t, store.Put(key, &Entry{ + Meta: Meta{EntryID: key, SourceHash: key, CreatedAt: base.Add(time.Duration(i) * time.Hour)}, + Proto: []byte(key), + })) + } + + _, oldExists, err := store.Get("old") + require.NoError(t, err) + assert.False(t, oldExists) + for _, key := range []string{"middle", "new"} { + _, exists, getErr := store.Get(key) + require.NoError(t, getErr) + assert.True(t, exists) + } +} + +func TestDiskStorePrunesToByteLimit(t *testing.T) { + store := NewBoundedDiskStore(t.TempDir(), 1200, 10, 1) + require.NoError(t, store.Put("old", &Entry{ + Meta: Meta{EntryID: "old", SourceHash: "old", CreatedAt: time.Unix(1, 0)}, + Proto: make([]byte, 512), + })) + require.NoError(t, store.Put("new", &Entry{ + Meta: Meta{EntryID: "new", SourceHash: "new", CreatedAt: time.Unix(2, 0)}, + Proto: make([]byte, 512), + })) + + _, oldExists, err := store.Get("old") + require.NoError(t, err) + assert.False(t, oldExists) + _, newExists, err := store.Get("new") + require.NoError(t, err) + assert.True(t, newExists) +} diff --git a/runtime/lua/code/cache/store.go b/runtime/lua/code/cache/store.go index cd3c89425..bde4dd8f5 100644 --- a/runtime/lua/code/cache/store.go +++ b/runtime/lua/code/cache/store.go @@ -30,6 +30,11 @@ type Deleter interface { Delete(key string) error } +// Pruner bounds retained cache generations according to the store policy. +type Pruner interface { + Prune() error +} + // Meta stores identifying information for an entry. type Meta struct { CreatedAt time.Time `json:"created_at"` diff --git a/runtime/lua/code/cache_fingerprint.go b/runtime/lua/code/cache_fingerprint.go index 61e963907..3eb52ec55 100644 --- a/runtime/lua/code/cache_fingerprint.go +++ b/runtime/lua/code/cache_fingerprint.go @@ -10,10 +10,11 @@ import ( "github.com/wippyai/go-lua/types/io" "github.com/wippyai/runtime/api/registry" + luaapi "github.com/wippyai/runtime/api/runtime/lua" "github.com/wippyai/runtime/runtime/lua/code/cache" ) -const cacheCompilerVersion = "lua-cache-v3" +const cacheCompilerVersion = "lua-cache-v4" // CacheCompilerVersion returns the cache compiler version string. func CacheCompilerVersion() string { @@ -75,6 +76,14 @@ func BuiltinManifestHash(manifests map[string]*io.Manifest) string { return hex.EncodeToString(h.Sum(nil)) } +// EffectiveMethod returns the method used by runtime compilation. +func EffectiveMethod(kind, method string) string { + if kind == luaapi.Library || kind == luaapi.LibraryBytecode { + return "" + } + return method +} + // CompileFingerprint computes the compile fingerprint for a node. func CompileFingerprint(entryID, kind, sourceHash, method string, deps []cache.DepFingerprint) string { self := cache.HashStrings("compile", cacheCompilerVersion, entryID, kind, method, sourceHash) diff --git a/runtime/lua/code/cache_fingerprint_test.go b/runtime/lua/code/cache_fingerprint_test.go index 9ad4dffd3..7c6daa49e 100644 --- a/runtime/lua/code/cache_fingerprint_test.go +++ b/runtime/lua/code/cache_fingerprint_test.go @@ -6,12 +6,19 @@ import ( "testing" "github.com/stretchr/testify/assert" + luaapi "github.com/wippyai/runtime/api/runtime/lua" "github.com/wippyai/runtime/runtime/lua/code/cache" ) func TestCacheCompilerVersion(t *testing.T) { v := CacheCompilerVersion() - assert.Equal(t, "lua-cache-v3", v) + assert.Equal(t, "lua-cache-v4", v) +} + +func TestEffectiveMethod(t *testing.T) { + assert.Empty(t, EffectiveMethod(luaapi.Library, "ignored")) + assert.Empty(t, EffectiveMethod(luaapi.LibraryBytecode, "ignored")) + assert.Equal(t, "handle", EffectiveMethod(luaapi.Function, "handle")) } func TestTypecheckConfigHash_Deterministic(t *testing.T) { diff --git a/runtime/lua/code/cache_runtime.go b/runtime/lua/code/cache_runtime.go index 96bc34f7f..465ac34b4 100644 --- a/runtime/lua/code/cache_runtime.go +++ b/runtime/lua/code/cache_runtime.go @@ -196,22 +196,6 @@ func (cm *Manager) compileFingerprintFromGraph(memGraph *MemoryGraph, id registr return fp, meta[id], nil } -func (cm *Manager) compileFingerprints(ids []registry.ID) map[registry.ID]string { - if len(ids) == 0 { - return nil - } - memo := make(map[registry.ID]string) - meta := make(map[registry.ID][]cache.DepMeta) - out := make(map[registry.ID]string, len(ids)) - for _, id := range ids { - fp, err := cm.compileFingerprintMemo(cm.memGraph, id, memo, meta) - if err == nil && fp != "" { - out[id] = fp - } - } - return out -} - func (cm *Manager) compileFingerprintMemo(memGraph *MemoryGraph, id registry.ID, memo map[registry.ID]string, meta map[registry.ID][]cache.DepMeta) (string, error) { if v, ok := memo[id]; ok { return v, nil @@ -260,22 +244,6 @@ func (cm *Manager) typecheckFingerprintFromGraph(memGraph *MemoryGraph, id regis return fp, meta[id], nil } -func (cm *Manager) typecheckFingerprints(ids []registry.ID) map[registry.ID]string { - if len(ids) == 0 { - return nil - } - memo := make(map[registry.ID]string) - meta := make(map[registry.ID][]cache.DepMeta) - out := make(map[registry.ID]string, len(ids)) - for _, id := range ids { - fp, err := cm.typecheckFingerprintMemo(cm.memGraph, id, memo, meta) - if err == nil && fp != "" { - out[id] = fp - } - } - return out -} - func (cm *Manager) typecheckFingerprintMemo(memGraph *MemoryGraph, id registry.ID, memo map[registry.ID]string, meta map[registry.ID][]cache.DepMeta) (string, error) { if v, ok := memo[id]; ok { return v, nil @@ -323,28 +291,6 @@ func (cm *Manager) refreshBuiltinHash() { cm.builtinHash = BuiltinManifestHash(manifests) } -func (cm *Manager) deleteCacheFingerprints(compileFPs, typecheckFPs map[registry.ID]string) { - if !cm.cacheAllowsWrite() { - return - } - deleter, ok := cm.cacheDeleter() - if !ok { - return - } - for _, fp := range compileFPs { - if fp == "" { - continue - } - _ = deleter.Delete(cm.compileCacheKey(fp)) - } - for _, fp := range typecheckFPs { - if fp == "" { - continue - } - _ = deleter.Delete(cm.typecheckCacheKey(fp)) - } -} - // CacheStore exposes the cache store (nil if disabled). func (cm *Manager) CacheStore() cache.Store { return cm.cacheStore @@ -364,3 +310,11 @@ func (cm *Manager) BuiltinManifestHash() string { func (cm *Manager) TypecheckConfigHash() string { return cm.typeCfgHash } + +// TypeCheckConfig returns the effective runtime type-check configuration. +func (cm *Manager) TypeCheckConfig() TypeCheckConfig { + if cm == nil || cm.typeChecker == nil { + return DefaultTypeCheckConfig() + } + return cm.typeChecker.config +} diff --git a/runtime/lua/code/cache_runtime_test.go b/runtime/lua/code/cache_runtime_test.go index c3e184d91..1d85a1817 100644 --- a/runtime/lua/code/cache_runtime_test.go +++ b/runtime/lua/code/cache_runtime_test.go @@ -3,14 +3,162 @@ package code import ( + "context" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/wippyai/go-lua/types/io" "github.com/wippyai/runtime/api/registry" api "github.com/wippyai/runtime/api/runtime/lua" + "github.com/wippyai/runtime/runtime/lua/code/cache" + "go.uber.org/zap" ) +func TestBuiltinNodeContentHashMatchesLintSeed(t *testing.T) { + initial := &api.ModuleDef{ + Name: "initial-module", + Types: func() *io.Manifest { return io.NewManifest("initial-module") }, + } + late := &api.ModuleDef{ + Name: "late-module", + Types: func() *io.Manifest { return io.NewManifest("late-module") }, + } + + cm, err := NewCodeManager(zap.NewNop(), &testEventBus{}, Config{Modules: []*api.ModuleDef{initial}}) + require.NoError(t, err) + + initialNode, err := cm.memGraph.GetNode(registry.NewID("", initial.Name)) + require.NoError(t, err) + lateNode := Node{ + ID: registry.NewID("", late.Name), + Kind: api.ModuleKind, + Module: late, + Manifest: late.Types(), + } + require.NoError(t, cm.AddNode(nil, lateNode, nil)) + cm.AddBuiltinType(late) + + for _, node := range []*Node{initialNode, mustGetNode(t, cm, lateNode.ID)} { + if got := nodeContentHash(node); got != cache.SourceHash("", "") { + t.Fatalf("builtin %s content hash = %q, want canonical lint seed %q", node.ID, got, cache.SourceHash("", "")) + } + } +} + +func TestContentAddressedCacheSurvivesGraphMutation(t *testing.T) { + typeCfg := DefaultTypeCheckConfig() + typeCfg.Enabled = true + typeCfg.Strict = true + cm, err := NewCodeManager(zap.NewNop(), &testEventBus{}, Config{ + TypeCheck: typeCfg, + Cache: cache.Config{ + Dir: t.TempDir(), + Enabled: true, + CompileEnabled: true, + TypecheckEnabled: true, + }, + }) + require.NoError(t, err) + + id := registry.NewID("test.cache", "entry") + ctx := context.Background() + require.NoError(t, cm.AddNode(ctx, Node{ID: id, Kind: api.Function, Source: `return "old"`, Method: "main"}, nil)) + compileFP, _, err := cm.compileFingerprint(id) + require.NoError(t, err) + typecheckFP, _, err := cm.typecheckFingerprint(id) + require.NoError(t, err) + _, err = cm.Compile(id, nil) + require.NoError(t, err) + + for _, key := range []string{cm.compileCacheKey(compileFP), cm.typecheckCacheKey(typecheckFP)} { + _, ok, getErr := cm.cacheStore.Get(key) + require.NoError(t, getErr) + require.True(t, ok) + } + + require.NoError(t, cm.UpdateNode(ctx, Node{ID: id, Kind: api.Function, Source: `return "new"`, Method: "main"}, nil)) + require.NoError(t, cm.DeleteNode(ctx, id)) + for _, key := range []string{cm.compileCacheKey(compileFP), cm.typecheckCacheKey(typecheckFP)} { + _, ok, getErr := cm.cacheStore.Get(key) + require.NoError(t, getErr) + require.True(t, ok) + } +} + +func TestRuntimeFingerprintsMatchLintBuiltinSeeds(t *testing.T) { + initial := &api.ModuleDef{ + Name: "initial-module", + Types: func() *io.Manifest { return io.NewManifest("initial-module") }, + } + late := &api.ModuleDef{ + Name: "late-module", + Types: func() *io.Manifest { return io.NewManifest("late-module") }, + } + typeCfg := TypeCheckConfig{Enabled: true, Strict: true} + cm, err := NewCodeManager(zap.NewNop(), &testEventBus{}, Config{ + Modules: []*api.ModuleDef{initial}, + TypeCheck: typeCfg, + }) + require.NoError(t, err) + + initialID := registry.NewID("", initial.Name) + lateID := registry.NewID("", late.Name) + userID := registry.NewID("app", "main") + require.NoError(t, cm.AddNode(nil, Node{ + ID: lateID, + Kind: api.ModuleKind, + Module: late, + Manifest: late.Types(), + }, nil)) + cm.AddBuiltinType(late) + require.NoError(t, cm.AddNode(nil, Node{ + ID: userID, + Kind: api.Function, + Source: "return initial + late", + Method: "main", + }, []Import{ + {ID: initialID, Alias: "initial"}, + {ID: lateID, Alias: "late"}, + })) + + compileFP, _, err := cm.compileFingerprint(userID) + require.NoError(t, err) + initialCompileFP := CompileFingerprint(initialID.String(), api.ModuleKind, cache.SourceHash("", ""), "", nil) + lateCompileFP := CompileFingerprint(lateID.String(), api.ModuleKind, cache.SourceHash("", ""), "", nil) + wantCompileFP := CompileFingerprint(userID.String(), api.Function, cache.SourceHash("return initial + late", "main"), "main", []cache.DepFingerprint{ + {Alias: "initial", ID: initialID.String(), Fingerprint: initialCompileFP}, + {Alias: "late", ID: lateID.String(), Fingerprint: lateCompileFP}, + }) + if compileFP != wantCompileFP { + t.Fatalf("runtime compile fingerprint = %q, lint formula = %q", compileFP, wantCompileFP) + } + + typeHash := TypecheckConfigHash(typeCfg) + builtinHash := BuiltinManifestHash(map[string]*io.Manifest{ + initial.Name: initial.Types(), + late.Name: late.Types(), + }) + typeFP, _, err := cm.typecheckFingerprint(userID) + require.NoError(t, err) + initialTypeFP := TypecheckFingerprint(initialID.String(), api.ModuleKind, cache.SourceHash("", ""), "", typeHash, builtinHash, nil) + lateTypeFP := TypecheckFingerprint(lateID.String(), api.ModuleKind, cache.SourceHash("", ""), "", typeHash, builtinHash, nil) + wantTypeFP := TypecheckFingerprint(userID.String(), api.Function, cache.SourceHash("return initial + late", "main"), "main", typeHash, builtinHash, []cache.DepFingerprint{ + {Alias: "initial", ID: initialID.String(), Fingerprint: initialTypeFP}, + {Alias: "late", ID: lateID.String(), Fingerprint: lateTypeFP}, + }) + if typeFP != wantTypeFP { + t.Fatalf("runtime typecheck fingerprint = %q, lint formula = %q", typeFP, wantTypeFP) + } +} + +func mustGetNode(t *testing.T, cm *Manager, id registry.ID) *Node { + t.Helper() + node, err := cm.memGraph.GetNode(id) + require.NoError(t, err) + return node +} + func TestCompileFingerprintCascades(t *testing.T) { cm, libNode, appID := setupFingerprintGraph(t) diff --git a/runtime/lua/code/compiler.go b/runtime/lua/code/compiler.go index 99d9e471f..1b1dcabba 100644 --- a/runtime/lua/code/compiler.go +++ b/runtime/lua/code/compiler.go @@ -8,7 +8,7 @@ import ( glua "github.com/wippyai/go-lua" "github.com/wippyai/runtime/api/registry" "github.com/wippyai/runtime/api/runtime/lua" - lru "github.com/wippyai/runtime/internal/cache" + "github.com/wippyai/runtime/runtime/lua/code/cache" ) // CompiledProto represents a compiled Lua prototype with its name @@ -30,114 +30,130 @@ type CompiledMain struct { Dependencies []CompiledProto } +// buildMemo shares graph-derived fingerprints across one complete build. +// Without this, deep dependency chains are traversed again for every node. +type buildMemo struct { + runtime map[registry.ID]string + compile map[registry.ID]string + compileMeta map[registry.ID][]cache.DepMeta + typecheck map[registry.ID]string + typecheckMeta map[registry.ID][]cache.DepMeta +} + +func newBuildMemo() *buildMemo { + return &buildMemo{ + runtime: make(map[registry.ID]string), + compile: make(map[registry.ID]string), + compileMeta: make(map[registry.ID][]cache.DepMeta), + typecheck: make(map[registry.ID]string), + typecheckMeta: make(map[registry.ID][]cache.DepMeta), + } +} + // CompileFn compiles a node against the graph snapshot used for the build. type CompileFn func(memGraph *MemoryGraph, node *Node) (*glua.FunctionProto, error) -type compiledProtoCacheKey struct { +type compileMemoFn func(memGraph *MemoryGraph, node *Node, memo *buildMemo) (*glua.FunctionProto, error) + +type retainedProtoKey struct { ID registry.ID Tag string } -type compiledMainCacheKey struct { +type retainedMainKey struct { ID registry.ID Tag string Options string } -// Compiler handles the compilation of Lua code and caches results +// Compiler retains compiled code until its owning registry nodes are invalidated. type Compiler struct { - protoCache *lru.Cache[compiledProtoCacheKey, *glua.FunctionProto] - mainCache *lru.Cache[compiledMainCacheKey, *CompiledMain] - protoByID map[registry.ID]map[compiledProtoCacheKey]struct{} - mainByID map[registry.ID]map[compiledMainCacheKey]struct{} - compileFn CompileFn - indexMu sync.Mutex + retainedProtos map[retainedProtoKey]*glua.FunctionProto + retainedMains map[retainedMainKey]*CompiledMain + protosByNode map[registry.ID]map[retainedProtoKey]struct{} + mainsByNode map[registry.ID]map[retainedMainKey]struct{} + compileFn CompileFn + compileMemoFn compileMemoFn + retainedMu sync.RWMutex } -// NewCompiler returns a new Compiler with caches -func NewCompiler( - compileFn CompileFn, - protoCacheCapacity int, - mainCacheCapacity int, -) *Compiler { - c := &Compiler{ - protoByID: make(map[registry.ID]map[compiledProtoCacheKey]struct{}), - mainByID: make(map[registry.ID]map[compiledMainCacheKey]struct{}), - compileFn: compileFn, - } +// NewCompiler returns a compiler with lifecycle-owned retained code. +func NewCompiler(compileFn CompileFn) *Compiler { + compiler := newCompiler() + compiler.compileFn = compileFn + return compiler +} - c.protoCache = lru.New[compiledProtoCacheKey, *glua.FunctionProto]( - lru.WithCapacity(protoCacheCapacity), - lru.WithOnEvict(func(key compiledProtoCacheKey, _ *glua.FunctionProto) { - c.removeProtoKey(key) - }), - ) - c.mainCache = lru.New[compiledMainCacheKey, *CompiledMain]( - lru.WithCapacity(mainCacheCapacity), - lru.WithOnEvict(func(key compiledMainCacheKey, _ *CompiledMain) { - c.removeMainKey(key) - }), - ) - - return c +func newCompilerWithMemo(compileFn compileMemoFn) *Compiler { + compiler := newCompiler() + compiler.compileMemoFn = compileFn + return compiler } -// getCompiledProto retrieves a node's compiled function prototype from cache or compiles it -func (c *Compiler) getCompiledProto(memGraph *MemoryGraph, node *Node, memo map[registry.ID]string) (*glua.FunctionProto, error) { +func newCompiler() *Compiler { + return &Compiler{ + retainedProtos: make(map[retainedProtoKey]*glua.FunctionProto), + retainedMains: make(map[retainedMainKey]*CompiledMain), + protosByNode: make(map[registry.ID]map[retainedProtoKey]struct{}), + mainsByNode: make(map[registry.ID]map[retainedMainKey]struct{}), + } +} + +// getCompiledProto retrieves retained code or compiles it for the active node version. +func (c *Compiler) getCompiledProto(memGraph *MemoryGraph, node *Node, memo *buildMemo) (*glua.FunctionProto, error) { if node.Kind == lua.ModuleKind { return nil, ErrModuleNotCompiled } - tag, err := runtimeFingerprintMemo(memGraph, node.ID, memo) + tag, err := runtimeFingerprintMemo(memGraph, node.ID, memo.runtime) if err != nil { return nil, err } - key := compiledProtoCacheKey{ID: node.ID, Tag: tag} + key := retainedProtoKey{ID: node.ID, Tag: tag} - if proto, ok := c.protoCache.Get(key); ok { + c.retainedMu.RLock() + proto, ok := c.retainedProtos[key] + c.retainedMu.RUnlock() + if ok { return proto, nil } - compiled, err := c.compileFn(memGraph, node) + var compiled *glua.FunctionProto + if c.compileMemoFn != nil { + compiled, err = c.compileMemoFn(memGraph, node, memo) + } else { + compiled, err = c.compileFn(memGraph, node) + } if err != nil { return nil, err } - _ = c.protoCache.Set(key, compiled) - c.recordProtoKey(key) - return compiled, nil + return c.retainProto(key, compiled), nil } -// Invalidate removes entries from both caches for the given IDs +// Invalidate releases retained code owned by the given registry nodes. func (c *Compiler) Invalidate(ids []registry.ID) { - c.indexMu.Lock() - protoKeys := make([]compiledProtoCacheKey, 0) - mainKeys := make([]compiledMainCacheKey, 0) + c.retainedMu.Lock() + defer c.retainedMu.Unlock() for _, id := range ids { - for key := range c.protoByID[id] { - protoKeys = append(protoKeys, key) + for key := range c.protosByNode[id] { + delete(c.retainedProtos, key) } - delete(c.protoByID, id) - for key := range c.mainByID[id] { - mainKeys = append(mainKeys, key) + delete(c.protosByNode, id) + for key := range c.mainsByNode[id] { + delete(c.retainedMains, key) } - delete(c.mainByID, id) - } - c.indexMu.Unlock() - - for _, key := range protoKeys { - c.protoCache.Delete(key) - } - for _, key := range mainKeys { - c.mainCache.Delete(key) + delete(c.mainsByNode, id) } } // SetProto injects a precompiled prototype into the cache. func (c *Compiler) SetProto(id registry.ID, tag string, proto *glua.FunctionProto) { - key := compiledProtoCacheKey{ID: id, Tag: tag} - _ = c.protoCache.Set(key, proto) - c.recordProtoKey(key) + key := retainedProtoKey{ID: id, Tag: tag} + c.retainedMu.Lock() + defer c.retainedMu.Unlock() + c.retainedProtos[key] = proto + c.recordProtoKeyLocked(key) } // Compile builds and compiles a main function and its dependencies @@ -150,18 +166,21 @@ func (c *Compiler) Compile( options = NewBuildOptions() } - memo := make(map[registry.ID]string) - tag, err := runtimeFingerprintMemo(memGraph, entrypoint, memo) + memo := newBuildMemo() + tag, err := runtimeFingerprintMemo(memGraph, entrypoint, memo.runtime) if err != nil { return nil, err } - key := compiledMainCacheKey{ + key := retainedMainKey{ ID: entrypoint, Tag: tag, Options: BuildOptionsFingerprint(options), } - if cached, ok := c.mainCache.Get(key); ok { + c.retainedMu.RLock() + cached, ok := c.retainedMains[key] + c.retainedMu.RUnlock() + if ok { return cached, nil } @@ -222,10 +241,7 @@ func (c *Compiler) Compile( compiled.Main = mainProto - _ = c.mainCache.Set(key, compiled) - c.recordMainKey(key) - - return compiled, nil + return c.retainMain(key, compiled), nil } func (c *Compiler) preloadModule(memGraph *MemoryGraph, pre Preload, compiled *CompiledMain) error { @@ -241,54 +257,38 @@ func (c *Compiler) preloadModule(memGraph *MemoryGraph, pre Preload, compiled *C return nil } -func (c *Compiler) recordProtoKey(key compiledProtoCacheKey) { - c.indexMu.Lock() - defer c.indexMu.Unlock() - - keys := c.protoByID[key.ID] - if keys == nil { - keys = make(map[compiledProtoCacheKey]struct{}) - c.protoByID[key.ID] = keys +func (c *Compiler) retainProto(key retainedProtoKey, proto *glua.FunctionProto) *glua.FunctionProto { + c.retainedMu.Lock() + defer c.retainedMu.Unlock() + if retained, ok := c.retainedProtos[key]; ok { + return retained } - keys[key] = struct{}{} + c.retainedProtos[key] = proto + c.recordProtoKeyLocked(key) + return proto } -func (c *Compiler) removeProtoKey(key compiledProtoCacheKey) { - c.indexMu.Lock() - defer c.indexMu.Unlock() - - keys := c.protoByID[key.ID] +func (c *Compiler) recordProtoKeyLocked(key retainedProtoKey) { + keys := c.protosByNode[key.ID] if keys == nil { - return - } - delete(keys, key) - if len(keys) == 0 { - delete(c.protoByID, key.ID) - } -} - -func (c *Compiler) recordMainKey(key compiledMainCacheKey) { - c.indexMu.Lock() - defer c.indexMu.Unlock() - - keys := c.mainByID[key.ID] - if keys == nil { - keys = make(map[compiledMainCacheKey]struct{}) - c.mainByID[key.ID] = keys + keys = make(map[retainedProtoKey]struct{}) + c.protosByNode[key.ID] = keys } keys[key] = struct{}{} } -func (c *Compiler) removeMainKey(key compiledMainCacheKey) { - c.indexMu.Lock() - defer c.indexMu.Unlock() - - keys := c.mainByID[key.ID] - if keys == nil { - return +func (c *Compiler) retainMain(key retainedMainKey, compiled *CompiledMain) *CompiledMain { + c.retainedMu.Lock() + defer c.retainedMu.Unlock() + if retained, ok := c.retainedMains[key]; ok { + return retained } - delete(keys, key) - if len(keys) == 0 { - delete(c.mainByID, key.ID) + c.retainedMains[key] = compiled + keys := c.mainsByNode[key.ID] + if keys == nil { + keys = make(map[retainedMainKey]struct{}) + c.mainsByNode[key.ID] = keys } + keys[key] = struct{}{} + return compiled } diff --git a/runtime/lua/code/manager.go b/runtime/lua/code/manager.go index 81d49882c..a76658047 100644 --- a/runtime/lua/code/manager.go +++ b/runtime/lua/code/manager.go @@ -108,16 +108,14 @@ type ( revision atomic.Uint64 invalidationSeq atomic.Uint64 invalidationWaitTimeout time.Duration - mutMu sync.Mutex + mutMu sync.RWMutex txMu sync.Mutex } // Config defines initialization parameters Config struct { - Cache cache.Config Modules []*api.ModuleDef - ProtoCacheSize int - MainCacheSize int + Cache cache.Config TypeCheck TypeCheckConfig InvalidationWaitTimeout time.Duration } @@ -125,13 +123,6 @@ type ( // NewCodeManager creates a new code manager instance func NewCodeManager(log *zap.Logger, bus event.Bus, cfg Config) (*Manager, error) { - if cfg.ProtoCacheSize <= 0 { - cfg.ProtoCacheSize = 5000 - } - - if cfg.MainCacheSize <= 0 { - cfg.MainCacheSize = 1000 - } if cfg.InvalidationWaitTimeout <= 0 { cfg.InvalidationWaitTimeout = DefaultInvalidationWaitTimeout } @@ -150,12 +141,14 @@ func NewCodeManager(log *zap.Logger, bus event.Bus, cfg Config) (*Manager, error invalidationWaitTimeout: cfg.InvalidationWaitTimeout, } if cacheCfg.Enabled { - cm.cacheStore = cache.NewDiskStore(cacheCfg.Dir) + cm.cacheStore = cache.NewBoundedDiskStore( + cacheCfg.Dir, cacheCfg.MaxBytes, cacheCfg.MaxEntries, cacheCfg.PruneInterval, + ) } // Create compiler with a callback that can access cm.memGraph for dependency manifests - cm.compiler = NewCompiler( - func(memGraph *MemoryGraph, node *Node) (*glua.FunctionProto, error) { + cm.compiler = newCompilerWithMemo( + func(memGraph *MemoryGraph, node *Node, memo *buildMemo) (*glua.FunctionProto, error) { var chunk []ast.Stmt var parsed bool parseOnce := func() error { @@ -176,9 +169,11 @@ func NewCodeManager(log *zap.Logger, bus event.Bus, cfg Config) (*Manager, error if typeChecker.IsEnabled() && node.Source != "" { var tcDeps []cache.DepMeta var tcFP string - if fingerprint, deps, err := cm.typecheckFingerprintFromGraph(memGraph, node.ID); err == nil { + if fingerprint, err := cm.typecheckFingerprintMemo( + memGraph, node.ID, memo.typecheck, memo.typecheckMeta, + ); err == nil { tcFP = fingerprint - tcDeps = deps + tcDeps = memo.typecheckMeta[node.ID] if manifest, cachedDiagnostics, ok := cm.loadTypecheckCache(node.ID, fingerprint); ok { node.Manifest = manifest cm.memGraph.SetManifestIfRevision(node.ID, node.Version.Revision, manifest) @@ -232,9 +227,11 @@ func NewCodeManager(log *zap.Logger, bus event.Bus, cfg Config) (*Manager, error var compileFP string var compileDeps []cache.DepMeta - if fingerprint, deps, err := cm.compileFingerprintFromGraph(memGraph, node.ID); err == nil { + if fingerprint, err := cm.compileFingerprintMemo( + memGraph, node.ID, memo.compile, memo.compileMeta, + ); err == nil { compileFP = fingerprint - compileDeps = deps + compileDeps = memo.compileMeta[node.ID] if proto, ok := cm.loadCompileCache(node.ID, fingerprint); ok { if node.Manifest != nil && len(proto.TypeInfo) == 0 { if data, err := node.Manifest.Encode(); err == nil { @@ -264,8 +261,6 @@ func NewCodeManager(log *zap.Logger, bus event.Bus, cfg Config) (*Manager, error return fnProto, nil }, - cfg.ProtoCacheSize, - cfg.MainCacheSize, ) // built-in modules @@ -275,7 +270,7 @@ func NewCodeManager(log *zap.Logger, bus event.Bus, cfg Config) (*Manager, error ID: registry.NewID("", info.Name), Kind: api.ModuleKind, Module: mod, - Version: cm.nextVersion(HashNode(&Node{Method: info.Name})), + Version: cm.nextVersion(cache.SourceHash("", "")), } if mod.Types != nil { node.Manifest = mod.Types() @@ -432,6 +427,8 @@ func (cm *Manager) Compile( entrypoint registry.ID, options *BuildOptions, ) (*CompiledMain, error) { + cm.mutMu.RLock() + defer cm.mutMu.RUnlock() return cm.compiler.Compile(cm.memGraph.Snapshot(), entrypoint, options) } @@ -474,7 +471,7 @@ func (cm *Manager) AddNode(_ context.Context, node Node, deps []Import) error { } // A delete followed by a create with the same registry ID must never reuse - // a previously compiled main/proto from the in-memory compiler caches. + // code retained for the previous registry node. cm.compiler.Invalidate([]registry.ID{node.ID}) cm.markTransactionAffected(nodePtr) @@ -492,17 +489,6 @@ func (cm *Manager) UpdateNode(_ context.Context, node Node, deps []Import) error } dependents, depErr := cm.memGraph.GetAllDependents(node.ID) - var oldCompileFPs map[registry.ID]string - var oldTypecheckFPs map[registry.ID]string - if cm.cacheAllowsWrite() { - invalidateIDs := make([]registry.ID, 0, len(dependents)+1) - invalidateIDs = append(invalidateIDs, node.ID) - for _, dep := range dependents { - invalidateIDs = append(invalidateIDs, dep.ID) - } - oldCompileFPs = cm.compileFingerprints(invalidateIDs) - oldTypecheckFPs = cm.typecheckFingerprints(invalidateIDs) - } // Eager compilation check: validate source code before updating if node.Source != "" && existing.Kind != api.ModuleKind { @@ -529,20 +515,16 @@ func (cm *Manager) UpdateNode(_ context.Context, node Node, deps []Import) error affectedNodes := cm.affectedNodes(nodePtr, dependents) cm.markTransactionAffected(affectedNodes...) - // Calculate all dependents for cache invalidation + // Calculate all dependents whose retained code is no longer valid. if depErr != nil { - cm.log.Warn("failed to get dependents for cache invalidation", + cm.log.Warn("failed to get dependents for retained code invalidation", zap.Stringer("node", &node.ID), zap.Error(depErr)) } - // Invalidate cache + // Release code owned by this node and its affected dependents. cm.compiler.Invalidate(nodeIDs(affectedNodes)) - if oldCompileFPs != nil || oldTypecheckFPs != nil { - cm.deleteCacheFingerprints(oldCompileFPs, oldTypecheckFPs) - } - return nil } @@ -576,13 +558,6 @@ func (cm *Manager) DeleteNode(_ context.Context, id registry.ID) error { dependents, depErr := cm.memGraph.GetAllDependents(id) - var oldCompileFPs map[registry.ID]string - var oldTypecheckFPs map[registry.ID]string - if cm.cacheAllowsWrite() { - oldCompileFPs = cm.compileFingerprints([]registry.ID{id}) - oldTypecheckFPs = cm.typecheckFingerprints([]registry.ID{id}) - } - if err := cm.memGraph.RemoveNode(id); err != nil { return NewRemoveNodeError(err) } @@ -598,10 +573,6 @@ func (cm *Manager) DeleteNode(_ context.Context, id registry.ID) error { cm.compiler.Invalidate(nodeIDs(affectedNodes)) - if oldCompileFPs != nil || oldTypecheckFPs != nil { - cm.deleteCacheFingerprints(oldCompileFPs, oldTypecheckFPs) - } - return nil } @@ -694,7 +665,7 @@ func (cm *Manager) GetTypeChecker() *TypeChecker { } // AddNodeWithProto adds a node with a precompiled prototype (for bytecode entries). -// The proto is injected directly into the compiler cache, bypassing source compilation. +// The proto is retained directly, bypassing source compilation. func (cm *Manager) AddNodeWithProto(_ context.Context, node Node, deps []Import, proto *glua.FunctionProto) error { nodePtr := &Node{ ID: node.ID, @@ -719,12 +690,11 @@ func (cm *Manager) AddNodeWithProto(_ context.Context, node Node, deps []Import, } } - // Clear any old compiled main/proto for this ID before registering a fresh - // bytecode node. SetProto below only replaces the proto cache; main cache - // must also be dropped. + // Release code owned by any previous node with this ID before registering + // the fresh bytecode node. cm.compiler.Invalidate([]registry.ID{node.ID}) - // Inject proto into compiler cache + // Retain the supplied proto for the active node version. if proto != nil { tag, err := runtimeFingerprintMemo(cm.memGraph, node.ID, make(map[registry.ID]string)) if err != nil { @@ -763,7 +733,7 @@ func (cm *Manager) UpdateNodeWithProto(_ context.Context, node Node, deps []Impo dependents, err := cm.memGraph.GetAllDependents(node.ID) if err != nil { - cm.log.Warn("failed to get dependents for cache invalidation", + cm.log.Warn("failed to get dependents for retained code invalidation", zap.Stringer("node", &node.ID), zap.Error(err)) } @@ -772,7 +742,7 @@ func (cm *Manager) UpdateNodeWithProto(_ context.Context, node Node, deps []Impo cm.markTransactionAffected(affectedNodes...) cm.compiler.Invalidate(nodeIDs(affectedNodes)) - // Inject updated proto into compiler cache + // Retain the supplied proto for the active node version. if proto != nil { tag, err := runtimeFingerprintMemo(cm.memGraph, node.ID, make(map[registry.ID]string)) if err != nil { diff --git a/runtime/lua/code/manager_test.go b/runtime/lua/code/manager_test.go index ede02b12f..8a7034342 100644 --- a/runtime/lua/code/manager_test.go +++ b/runtime/lua/code/manager_test.go @@ -47,32 +47,19 @@ func (b *testEventBus) Unsubscribe(_ context.Context, _ event.SubscriberID) { func TestNewCodeManager(t *testing.T) { tests := []struct { - name string - modules []*api.ModuleDef - protoCacheSize int - mainCacheSize int - expectErr bool + name string + modules []*api.ModuleDef + expectErr bool }{ { - name: "Default cache sizes", - modules: []*api.ModuleDef{{Name: "test"}}, - protoCacheSize: 0, - mainCacheSize: 0, - expectErr: false, - }, - { - name: "Custom cache sizes", - modules: []*api.ModuleDef{{Name: "test"}}, - protoCacheSize: 100, - mainCacheSize: 50, - expectErr: false, + name: "With modules", + modules: []*api.ModuleDef{{Name: "test"}}, + expectErr: false, }, { - name: "No modules", - modules: []*api.ModuleDef{}, - protoCacheSize: 0, - mainCacheSize: 0, - expectErr: false, + name: "No modules", + modules: []*api.ModuleDef{}, + expectErr: false, }, } @@ -80,11 +67,7 @@ func TestNewCodeManager(t *testing.T) { t.Run(tt.name, func(t *testing.T) { logger := zap.NewNop() bus := &testEventBus{} - cfg := Config{ - Modules: tt.modules, - ProtoCacheSize: tt.protoCacheSize, - MainCacheSize: tt.mainCacheSize, - } + cfg := Config{Modules: tt.modules} cm, err := NewCodeManager(logger, bus, cfg) if tt.expectErr { @@ -551,10 +534,7 @@ func TestManager_UpdateNode(t *testing.T) { func TestManager_UpdateNodeFailureLeavesGraphUnchanged(t *testing.T) { logger := zap.NewNop() bus := &testEventBus{} - cm, err := NewCodeManager(logger, bus, Config{ - ProtoCacheSize: 8, - MainCacheSize: 8, - }) + cm, err := NewCodeManager(logger, bus, Config{}) require.NoError(t, err) ctx := context.Background() @@ -695,13 +675,10 @@ func TestManager_Compile(t *testing.T) { } } -func TestManager_DeleteThenAddSameIDInvalidatesCompileCaches(t *testing.T) { +func TestManager_DeleteThenAddSameIDReleasesRetainedCode(t *testing.T) { logger := zap.NewNop() bus := &testEventBus{} - cm, err := NewCodeManager(logger, bus, Config{ - ProtoCacheSize: 8, - MainCacheSize: 8, - }) + cm, err := NewCodeManager(logger, bus, Config{}) require.NoError(t, err) id := registry.NewID("app.replay", "same_id") @@ -735,15 +712,27 @@ func TestManager_DeleteThenAddSameIDInvalidatesCompileCaches(t *testing.T) { require.Equal(t, "good", executeCompiledString(t, second.Main)) require.NotSame(t, first, second) require.NotSame(t, first.Main, second.Main) + + cm.compiler.retainedMu.RLock() + protoCount := len(cm.compiler.protosByNode[id]) + mainCount := len(cm.compiler.mainsByNode[id]) + cm.compiler.retainedMu.RUnlock() + require.Equal(t, 1, protoCount) + require.Equal(t, 1, mainCount) + + require.NoError(t, cm.DeleteNode(ctx, id)) + cm.compiler.retainedMu.RLock() + _, hasProto := cm.compiler.protosByNode[id] + _, hasMain := cm.compiler.mainsByNode[id] + cm.compiler.retainedMu.RUnlock() + require.False(t, hasProto) + require.False(t, hasMain) } func TestManager_SameIDRecreateUsesNewRevisionEvenWithoutManualInvalidation(t *testing.T) { logger := zap.NewNop() bus := &testEventBus{} - cm, err := NewCodeManager(logger, bus, Config{ - ProtoCacheSize: 8, - MainCacheSize: 8, - }) + cm, err := NewCodeManager(logger, bus, Config{}) require.NoError(t, err) id := registry.NewID("app.replay", "revision_tag") @@ -778,10 +767,7 @@ func TestManager_SameIDRecreateUsesNewRevisionEvenWithoutManualInvalidation(t *t func TestManager_UpdateInvalidatesDependentMainCacheByFingerprint(t *testing.T) { logger := zap.NewNop() bus := &testEventBus{} - cm, err := NewCodeManager(logger, bus, Config{ - ProtoCacheSize: 8, - MainCacheSize: 8, - }) + cm, err := NewCodeManager(logger, bus, Config{}) require.NoError(t, err) ctx := context.Background() @@ -822,10 +808,7 @@ func TestManager_UpdateInvalidatesDependentMainCacheByFingerprint(t *testing.T) func TestBuildOptionsFingerprintSeparatesMainCache(t *testing.T) { logger := zap.NewNop() bus := &testEventBus{} - cm, err := NewCodeManager(logger, bus, Config{ - ProtoCacheSize: 8, - MainCacheSize: 8, - }) + cm, err := NewCodeManager(logger, bus, Config{}) require.NoError(t, err) id := registry.NewID("app.replay", "options") @@ -852,10 +835,7 @@ func TestBuildOptionsFingerprintSeparatesMainCache(t *testing.T) { func TestManager_ConcurrentCompileAndUpdate(t *testing.T) { logger := zap.NewNop() bus := &testEventBus{} - cm, err := NewCodeManager(logger, bus, Config{ - ProtoCacheSize: 16, - MainCacheSize: 16, - }) + cm, err := NewCodeManager(logger, bus, Config{}) require.NoError(t, err) id := registry.NewID("app.race", "compile_update") @@ -918,6 +898,15 @@ func TestManager_ConcurrentCompileAndUpdate(t *testing.T) { for err := range errCh { require.NoError(t, err) } + + _, err = cm.Compile(id, nil) + require.NoError(t, err) + cm.compiler.retainedMu.RLock() + protoCount := len(cm.compiler.protosByNode[id]) + mainCount := len(cm.compiler.mainsByNode[id]) + cm.compiler.retainedMu.RUnlock() + require.Equal(t, 1, protoCount, "only the live revision may remain retained") + require.Equal(t, 1, mainCount, "only the live revision may remain retained") } func executeCompiledString(t *testing.T, proto *glua.FunctionProto) string { @@ -1212,10 +1201,7 @@ func TestManager_AddNodeWithProto_CompileUsesProto(t *testing.T) { func TestManager_AddNodeWithProtoSameIDRecreateUsesNewRevision(t *testing.T) { logger := zap.NewNop() bus := &testEventBus{} - cm, err := NewCodeManager(logger, bus, Config{ - ProtoCacheSize: 8, - MainCacheSize: 8, - }) + cm, err := NewCodeManager(logger, bus, Config{}) require.NoError(t, err) id := registry.NewID("app.replay", "bytecode_same_id") diff --git a/runtime/lua/code/typecheck.go b/runtime/lua/code/typecheck.go index e8191bcbf..3c2510d20 100644 --- a/runtime/lua/code/typecheck.go +++ b/runtime/lua/code/typecheck.go @@ -3,6 +3,8 @@ package code import ( + "sync" + "github.com/wippyai/go-lua/compiler/ast" "github.com/wippyai/go-lua/compiler/check" "github.com/wippyai/go-lua/compiler/check/hooks" @@ -82,6 +84,7 @@ type TypeChecker struct { baseHookOptions []check.Option hookOptions []check.Option config TypeCheckConfig + checkMu sync.Mutex } // NewTypeChecker creates a configured type checker. @@ -160,12 +163,26 @@ func NewTypeChecker(cfg TypeCheckConfig, builtinMods []*api.ModuleDef) *TypeChec // CheckParsed performs type checking on a parsed AST with provided imports func (tc *TypeChecker) CheckParsed(chunk []ast.Stmt, entryID string, imports map[string]*io.Manifest) (*io.Manifest, []diag.Diagnostic) { + tc.checkMu.Lock() + defer tc.checkMu.Unlock() + + previous := make(map[string]*io.Manifest, len(imports)) // Connect imports to database for alias, manifest := range imports { + previous[alias] = tc.db.Manifest(alias) if manifest != nil { tc.db.Connect(alias, manifest) } } + defer func() { + for alias, manifest := range previous { + if manifest == nil { + tc.db.Disconnect(alias) + continue + } + tc.db.Connect(alias, manifest) + } + }() // Check the chunk sess := tc.checker.CheckChunk(chunk, entryID) @@ -354,6 +371,8 @@ func (tc *TypeChecker) ClearCache() { if tc == nil || tc.checker == nil { return } + tc.checkMu.Lock() + defer tc.checkMu.Unlock() tc.checker.ClearCache() } diff --git a/runtime/lua/code/typecheck_test.go b/runtime/lua/code/typecheck_test.go index 6e2d70c27..03abe47aa 100644 --- a/runtime/lua/code/typecheck_test.go +++ b/runtime/lua/code/typecheck_test.go @@ -13,6 +13,7 @@ import ( "github.com/wippyai/go-lua/types/io" "github.com/wippyai/go-lua/types/query/core" "github.com/wippyai/go-lua/types/typ" + api "github.com/wippyai/runtime/api/runtime/lua" base64mod "github.com/wippyai/runtime/runtime/lua/modules/base64" ) @@ -356,6 +357,33 @@ return x } } +func TestTypeChecker_CheckImportsAreRequestScoped(t *testing.T) { + tc := NewTypeChecker(DefaultTypeCheckConfig(), nil) + dependency := io.NewManifest("dependency") + + _, _, err := tc.Check("return {}", "first.lua", map[string]*io.Manifest{ + "dependency": dependency, + }) + require.NoError(t, err) + require.Nil(t, tc.db.Manifest("dependency")) +} + +func TestTypeChecker_CheckRestoresShadowedBuiltinManifest(t *testing.T) { + builtin := io.NewManifest("builtin") + module := &api.ModuleDef{ + Name: "dependency", + Types: func() *io.Manifest { return builtin }, + } + tc := NewTypeChecker(DefaultTypeCheckConfig(), []*api.ModuleDef{module}) + override := io.NewManifest("override") + + _, _, err := tc.Check("return {}", "first.lua", map[string]*io.Manifest{ + "dependency": override, + }) + require.NoError(t, err) + require.Same(t, builtin, tc.db.Manifest("dependency")) +} + // TestTypeChecker_ManifestTypeAlias tests that type aliases from manifests // are structurally equivalent to their underlying types. func TestTypeChecker_ManifestTypeAlias(t *testing.T) { diff --git a/runtime/lua/engine/factory_test.go b/runtime/lua/engine/factory_test.go index a6e6827c3..53505cbf5 100644 --- a/runtime/lua/engine/factory_test.go +++ b/runtime/lua/engine/factory_test.go @@ -71,11 +71,7 @@ var factoryWorkflowAllowedIDs = []registry.ID{ func setupFactoryCodeManager(t *testing.T) *code.Manager { t.Helper() log := zap.NewNop() - cm, err := code.NewCodeManager(log, nil, code.Config{ - Modules: nil, - ProtoCacheSize: 100, - MainCacheSize: 100, - }) + cm, err := code.NewCodeManager(log, nil, code.Config{Modules: nil}) if err != nil { t.Fatalf("failed to create code manager: %v", err) } diff --git a/service/temporal/worker/send_test.go b/service/temporal/worker/send_test.go index da06760a4..c281668ba 100644 --- a/service/temporal/worker/send_test.go +++ b/service/temporal/worker/send_test.go @@ -93,9 +93,7 @@ func TestWorkerSend_Integration(t *testing.T) { bus := eventbus.NewBus() codeManager, err := code.NewCodeManager(logger, nil, code.Config{ - Modules: []*luaapi.ModuleDef{timemod.Module, processmod.Module}, - ProtoCacheSize: 100, - MainCacheSize: 100, + Modules: []*luaapi.ModuleDef{timemod.Module, processmod.Module}, }) require.NoError(t, err) diff --git a/service/temporal/workflow/integration_test.go b/service/temporal/workflow/integration_test.go index 1c85c2942..6cfae8c19 100644 --- a/service/temporal/workflow/integration_test.go +++ b/service/temporal/workflow/integration_test.go @@ -70,11 +70,7 @@ func newWorkflowTestFixture(t *testing.T, opts workflowTestOpts) *workflowTestFi logger := zap.NewNop() bus := eventbus.NewBus() - codeManager, err := code.NewCodeManager(logger, nil, code.Config{ - Modules: opts.modules, - ProtoCacheSize: 100, - MainCacheSize: 100, - }) + codeManager, err := code.NewCodeManager(logger, nil, code.Config{Modules: opts.modules}) require.NoError(t, err) processFactory := engine.NewProcessFactory(codeManager) diff --git a/system/registry/registry.go b/system/registry/registry.go index 7a9b1f9ca..a45e412da 100644 --- a/system/registry/registry.go +++ b/system/registry/registry.go @@ -760,6 +760,10 @@ func (r *Reg) collectBackwardChangesets(path []registry.Version, targetVersion r // For v0 (empty history): applies baseline directly // For v1+: replays changesets v1..targetVersion on top of baseline, then applies final state once func (r *Reg) LoadState(ctx context.Context, baseline registry.State, targetVersion registry.Version) error { + if registry.DependencyAccessFromContext(ctx) == registry.DependencyAccessUnspecified { + ctx = registry.WithDependencyAccess(ctx, registry.DependencyAccessVerifiedOffline) + } + r.applyMu.Lock() defer r.applyMu.Unlock() diff --git a/system/registry/registry_hardening_test.go b/system/registry/registry_hardening_test.go index 9c5d2b2d1..3fa4d2bcc 100644 --- a/system/registry/registry_hardening_test.go +++ b/system/registry/registry_hardening_test.go @@ -148,6 +148,34 @@ func hardeningResolution() *regapi.DependencyResolution { }).Canonical() } +func TestLoadStateDefaultsDependencyAccessToVerifiedOffline(t *testing.T) { + for _, test := range []struct { + ctx context.Context + name string + want regapi.DependencyAccess + }{ + {name: "default restore", ctx: context.Background(), want: regapi.DependencyAccessVerifiedOffline}, + { + name: "explicit online migration", + ctx: regapi.WithDependencyAccess(context.Background(), regapi.DependencyAccessOnline), + want: regapi.DependencyAccessOnline, + }, + } { + t.Run(test.name, func(t *testing.T) { + seen := regapi.DependencyAccessUnspecified + directive := hardeningDirective{expand: func(ctx context.Context, _ regapi.Operation, _ regapi.State) (regapi.DirectiveResult, error) { + seen = regapi.DependencyAccessFromContext(ctx) + return regapi.DirectiveResult{}, nil + }} + reg := NewRegistry(memory.New(), &hardeningRunner{}, topology.NewStateBuilder(zap.NewNop(), nil), nil, zap.NewNop(), + WithKindDirective(regapi.NamespaceDependency, directive)) + dep := regapi.Entry{ID: regapi.NewID("app.deps", "module"), Kind: regapi.NamespaceDependency} + require.NoError(t, reg.LoadState(test.ctx, regapi.State{dep}, version.New(0))) + require.Equal(t, test.want, seen) + }) + } +} + func TestApplyVersion_SetHeadFailureCompensatesRuntimeAndEffects(t *testing.T) { ctx := context.Background() v0 := version.New(0)