From a55fd4bcf6af48792fb910f822cbc588a638b8be Mon Sep 17 00:00:00 2001 From: stefanwalcz Date: Mon, 29 Jun 2026 22:44:24 +0200 Subject: [PATCH] fix(agent): self-correct when the model calls an unknown tool When the LLM selects a tool name that matches no available action (common with local models, which sometimes hallucinate tool names), the tool-call callback looked the name up via allActions.Find, got nil and fell through: execution proceeded with a nil action, yielding an empty result and -- when an observer is attached -- a nil-pointer dereference at chosenAction.Definition(). The model received no signal to recover and would typically repeat the invalid call or give up. Use cogito's existing ToolCallDecision.Adjustment mechanism to feed the valid tool list back so the model re-selects an existing tool. The correction is bounded (max 3 attempts per tool name); past that the call is skipped to avoid an unbounded re-selection loop. Built-in control verbs (stop/send_message/update_state) are dispatched by name and are excluded. The observer creation path is also guarded against a nil action. Adds a hermetic unit test for the decision logic (no live LLM required). Assisted-by: Claude:claude-opus-4-8 [Claude Code] Signed-off-by: stefanwalcz --- core/agent/agent.go | 72 ++++++++++++++++++++++++++-- core/agent/unknown_tool_test.go | 84 +++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 3 deletions(-) create mode 100644 core/agent/unknown_tool_test.go diff --git a/core/agent/agent.go b/core/agent/agent.go index 115c7c41..d0cdf220 100644 --- a/core/agent/agent.go +++ b/core/agent/agent.go @@ -747,6 +747,36 @@ func (a *Agent) filterJob(job *types.Job) (ok bool, err error) { return failedBy == "" && (!hasTriggers || triggeredBy != ""), nil } +// correctUnknownToolCall decides how to handle a tool call whose name does not +// match any available action. corrected is false for the built-in control verbs +// (stop/send_message/update_state), which are dispatched by name elsewhere and +// need no correction. Otherwise it returns an Adjustment that feeds the valid +// tool list back to the model so it can self-correct -- up to max attempts per +// tool name, after which the call is skipped to avoid an unbounded re-selection +// loop. attempts is mutated to track per-tool-name correction counts. +func correctUnknownToolCall(name string, available []string, attempts map[string]int, max int) (decision cogito.ToolCallDecision, corrected bool) { + switch name { + case action.StopActionName, action.ConversationActionName, action.StateActionName: + return cogito.ToolCallDecision{}, false + } + + if attempts[name] < max { + attempts[name]++ + return cogito.ToolCallDecision{ + Approved: true, + Adjustment: fmt.Sprintf( + "The tool %q does not exist and cannot be called. "+ + "You must call one of the available tools, using its exact name: %s. "+ + "Re-issue your request using one of these tools.", + name, strings.Join(available, ", ")), + }, true + } + + // Repeated hallucination: skip the bad call instead of looping forever or + // aborting the whole run. + return cogito.ToolCallDecision{Skip: true}, true +} + // replyWithToolCall handles user-defined actions by recording the action state without setting Response func (a *Agent) replyWithToolCall(job *types.Job, conv []openai.ChatCompletionMessage, params types.ActionParams, chosenAction types.Action, reasoning string) { // Record the action state so the webui can detect this is a user-defined action @@ -1044,6 +1074,13 @@ func (a *Agent) consumeJob(job *types.Job, role string) { var observables = make(map[string]*types.Observable) + // Bounds the number of self-correction nudges we send when the model + // invokes a tool that does not exist, so a model that keeps hallucinating + // tool names cannot loop forever (cogito re-runs tool selection on every + // Adjustment). Keyed by tool name; once exhausted the bad call is skipped. + const maxToolCorrectionAttempts = 3 + toolCorrectionAttempts := make(map[string]int) + cogitoOpts := []cogito.Option{ cogito.WithMCPs(a.mcpSessions...), cogito.WithTools( @@ -1151,6 +1188,30 @@ func (a *Agent) consumeJob(job *types.Job, role string) { xlog.Debug("Action found", "action", chosenAction) + // Guard against hallucinated / unavailable tool names. Models + // (local ones in particular) sometimes invoke a tool that was + // never offered. Without this the run proceeds with a nil action + // -- producing an empty result and a nil-pointer dereference in + // the observer/decision path below -- and the model receives no + // signal to recover. Feed the valid tool list back via cogito's + // Adjustment mechanism so the model re-selects an existing tool. + if chosenAction == nil { + available := make([]string, 0, len(allActions)) + for _, act := range allActions { + available = append(available, act.Definition().Name.String()) + } + if decision, corrected := correctUnknownToolCall(tc.Name, available, toolCorrectionAttempts, maxToolCorrectionAttempts); corrected { + if decision.Skip { + xlog.Warn("LLM repeatedly selected unknown tools, skipping call", + "tool", tc.Name, "available", available) + } else { + xlog.Warn("LLM selected an unknown tool, requesting self-correction", + "tool", tc.Name, "attempt", toolCorrectionAttempts[tc.Name], "available", available) + } + return decision + } + } + if chosenAction != nil && types.IsActionUserDefined(chosenAction) { xlog.Debug("User-defined action chosen, returning tool call", "action", chosenAction.Definition().Name) a.replyWithToolCall(job, conv, tc.Arguments, chosenAction, tc.Reasoning) @@ -1178,14 +1239,19 @@ func (a *Agent) consumeJob(job *types.Job, role string) { obs.Name = "decision" obs.ParentID = job.Obs.ID obs.Icon = "brain" - obs.Creation = &types.Creation{ + creation := &types.Creation{ ChatCompletionRequest: &openai.ChatCompletionRequest{ Model: a.options.LLMAPI.Model, Messages: conv, }, - FunctionDefinition: chosenAction.Definition().ToFunctionDefinition(), - FunctionParams: types.ActionParams(tc.Arguments), + FunctionParams: types.ActionParams(tc.Arguments), + } + // chosenAction can be nil for a built-in control verb that + // is handled by name below but absent from allActions. + if chosenAction != nil { + creation.FunctionDefinition = chosenAction.Definition().ToFunctionDefinition() } + obs.Creation = creation a.observer.Update(*obs) observables[tc.ID] = obs diff --git a/core/agent/unknown_tool_test.go b/core/agent/unknown_tool_test.go new file mode 100644 index 00000000..52aafa67 --- /dev/null +++ b/core/agent/unknown_tool_test.go @@ -0,0 +1,84 @@ +package agent + +import ( + "strings" + "testing" + + "github.com/mudler/LocalAGI/core/action" +) + +// TestCorrectUnknownToolCall covers the self-correction logic used when the +// model invokes a tool name that matches no available action. +func TestCorrectUnknownToolCall(t *testing.T) { + available := []string{"openregister_search", "openregister_company"} + const max = 3 + + t.Run("built-in control verbs are not treated as unknown", func(t *testing.T) { + for _, name := range []string{action.StopActionName, action.ConversationActionName, action.StateActionName} { + attempts := map[string]int{} + if _, corrected := correctUnknownToolCall(name, available, attempts, max); corrected { + t.Fatalf("control verb %q should not be corrected", name) + } + if len(attempts) != 0 { + t.Fatalf("control verb %q should not consume a correction attempt", name) + } + } + }) + + t.Run("unknown tool yields an adjustment listing the available tools", func(t *testing.T) { + attempts := map[string]int{} + decision, corrected := correctUnknownToolCall("list_collections", available, attempts, max) + if !corrected { + t.Fatal("unknown tool should be corrected") + } + if !decision.Approved { + t.Fatal("adjustment must keep Approved=true, otherwise cogito aborts the run instead of re-querying") + } + if decision.Skip { + t.Fatal("first attempts should adjust, not skip") + } + if decision.Adjustment == "" { + t.Fatal("expected a non-empty adjustment") + } + if !strings.Contains(decision.Adjustment, "list_collections") { + t.Errorf("adjustment should name the offending tool, got: %q", decision.Adjustment) + } + for _, name := range available { + if !strings.Contains(decision.Adjustment, name) { + t.Errorf("adjustment should list available tool %q, got: %q", name, decision.Adjustment) + } + } + if attempts["list_collections"] != 1 { + t.Errorf("expected attempt counter 1, got %d", attempts["list_collections"]) + } + }) + + t.Run("repeated hallucination is skipped after max attempts", func(t *testing.T) { + attempts := map[string]int{} + for i := 0; i < max; i++ { + decision, _ := correctUnknownToolCall("ghost_tool", available, attempts, max) + if decision.Skip { + t.Fatalf("attempt %d should still adjust, not skip", i+1) + } + } + decision, corrected := correctUnknownToolCall("ghost_tool", available, attempts, max) + if !corrected { + t.Fatal("over-limit unknown tool should still be handled") + } + if !decision.Skip { + t.Fatal("after max attempts the call must be skipped to avoid an unbounded loop") + } + if decision.Approved { + t.Fatal("a skipped call should not also be approved") + } + }) + + t.Run("attempt counters are per-tool-name", func(t *testing.T) { + attempts := map[string]int{} + correctUnknownToolCall("tool_a", available, attempts, max) + correctUnknownToolCall("tool_b", available, attempts, max) + if attempts["tool_a"] != 1 || attempts["tool_b"] != 1 { + t.Fatalf("counters should be independent per name, got %v", attempts) + } + }) +}