Add an AI Provider (claude) and use it to resolve local patch merge conflicts

.flake8

@@ -1,2 +1,3 @@
 [flake8]
 ignore = E501,E402,E275
+extend-exclude = .venv,venv

apis/phabricator.py

@@ -37,7 +37,7 @@ def __init__(self, config):
             self.url = config['url']
 
     @logEntryExit
-    def submit_patches(self, bug_id, has_patches):
+    def submit_patches(self, bug_id, num_commits):
         phab_revisions = []
 
         @retry
@@ -69,19 +69,23 @@ def submit_to_phabricator(rev_id, retry_attempt=None):
 
             return phab_revision
 
-        # arc diff will squash all commits into a single commit, so we need to jump through some hoops.
-        # Conceptually, we are only commiting the top-most commit in the repo (and not any subsequent commits)
-        # If we have two commits, we'll go backwards and grab only the first commit, then go back to tip
-        if has_patches:
-            # Checkout to the first patch
-            self.run(["hg", "checkout", "tip^"])
-            # Tell phabricator to submit from the base to the current working tree
+        # arc diff will squash everything from the base to the working parent into a
+        # single revision, so to submit each commit as its own revision we walk the
+        # stack from the bottom-most commit upward, submitting one commit at a time.
+        # (num_commits is 1 for a plain vendor, 2 when there are local patches, and 3
+        # when the AI added a commit resolving patch conflicts.)
+        if num_commits > 1:
+            # Checkout to the bottom-most commit we're submitting.
+            self.run(["hg", "checkout", "tip~%d" % (num_commits - 1)])
+            # Submit it as the diff from the base to the current working tree.
             phab_revisions.append(submit_to_phabricator(""))
-            # Ask hg to evolve the original second patch on top of the rewritten first patch
-            self.run(["hg", "next"])
-
-        # Submit only a single patch
-        phab_revisions.append(submit_to_phabricator("tip^"))
+            # Then evolve up one commit at a time, submitting each on its own.
+            for _ in range(num_commits - 1):
+                self.run(["hg", "next"])
+                phab_revisions.append(submit_to_phabricator(".^"))
+        else:
+            # Submit only the single (vendoring) commit.
+            phab_revisions.append(submit_to_phabricator("tip^"))
 
         # Chain revisions together if needed
         @retry

automation.py

@@ -14,6 +14,7 @@
 from components.libraryprovider import LibraryProvider
 from components.mach_vendor import VendorProvider
 from components.bugzilla import BugzillaProvider
+from components.aiprovider import AIProvider
 from components.scmprovider import SCMProvider
 from components.hg import MercurialProvider, reset_repository
 from apis.taskcluster import TaskclusterProvider
@@ -32,6 +33,7 @@
     'Taskcluster': TaskclusterProvider,
     'Phabricator': PhabricatorProvider,
     'SCM': SCMProvider,
+    'AI': AIProvider,
     'VendorTaskRunner': VendorTaskRunner,
     'CommitAlertTaskRunner': CommitAlertTaskRunner
 }
@@ -123,6 +125,7 @@ def getOr(name):
                 'taskclusterProvider': getOr('Taskcluster'),
                 'phabricatorProvider': getOr('Phabricator'),
                 'scmProvider': getOr('SCM'),
+                'aiProvider': getOr('AI'),
             })
             # Step 6
             self.runOnProviders(lambda x: x.update_config(additional_config))

components/aiprovider.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import json
+import shutil
+import tempfile
+
+from components.utilities import load_prompt
+from components.logging import logEntryExit, LogLevel
+from components.providerbase import BaseProvider, INeedsCommandProvider, INeedsLoggingProvider
+
+
+CONFLICT_RESOLUTION_RESULT_FILE = "conflict_resolution_result.json"
+
+
+class AIResult:
+    def __init__(self, success, text, session_id=None):
+        self.success = success
+        self.text = text
+        self.session_id = session_id
+
+
+class AIProvider(BaseProvider, INeedsCommandProvider, INeedsLoggingProvider):
+    """
+    Invokes an AI coding agent headlessly to perform a task described by a
+    prompt, and returns its result. Currently backed by the Claude Code CLI
+    (`claude -p`).
+
+    The prompt and system prompt are written to files rather than passed on the
+    command line, so we never hit command-line length limits: the prompt is fed
+    on stdin (claude reads the prompt from stdin when given no positional
+    prompt) and the system prompt is passed with --append-system-prompt-file.
+    """
+
+    def __init__(self, config):
+        self.model = config.get('model', 'claude-opus-4-8')
+        self.max_turns = config.get('max-turns', 30)
+        self.timeout = config.get('timeout', 60 * 60)
+        # API key for the AI CLI, supplied through the config dictionary (like
+        # the Database password and Bugzilla apikey). Passed to the CLI via the
+        # environment variable it expects rather than on the command line.
+        self.apikey = config.get('apikey', None)
+        # For debugging: a directory into which each invocation's prompt, system
+        # prompt, and the CLI's raw stdout/stderr are written. None disables it.
+        self.debug_output_dir = config.get('debug-output-dir', None)
+        self._invocation_count = 0
+        # A label (library + job id) folded into debug filenames, set per call.
+        self._debug_label = ""
+
+    @logEntryExit
+    def _run_prompt(self, prompt, system_prompt=None, cwd=None):
+        tmpdir = tempfile.mkdtemp(prefix="updatebot-claude-")
+        try:
+            prompt_path = os.path.join(tmpdir, "prompt.txt")
+            with open(prompt_path, "w") as f:
+                f.write(prompt)
+
+            args = ["claude", "-p",
+                    "--output-format", "json",
+                    "--permission-mode", "bypassPermissions",
+                    "--model", self.model,
+                    "--max-turns", str(self.max_turns)]
+
+            if system_prompt:
+                system_path = os.path.join(tmpdir, "system_prompt.txt")
+                with open(system_path, "w") as f:
+                    f.write(system_prompt)
+                args += ["--append-system-prompt-file", system_path]
+
+            env = {"ANTHROPIC_API_KEY": self.apikey} if self.apikey else None
+
+            # claude returns is_error (and a non-zero exit) on failure but still
+            # prints the JSON result we want to read, so don't let run() raise.
+            ret = self.run(args, shell=False, clean_return=False, cwd=cwd,
+                           stdin_path=prompt_path, timeout=self.timeout, env=env)
+            self._write_debug_output(prompt, system_prompt, ret)
+            return self._parse_result(ret)
+        finally:
+            shutil.rmtree(tmpdir, ignore_errors=True)
+
+    def _write_debug_output(self, prompt, system_prompt, ret):
+        # When debug-output-dir is configured, dump everything about this
+        # invocation so a live run can be inspected after the fact (the CLI
+        # buffers its JSON until completion, so nothing is available mid-run).
+        if not self.debug_output_dir:
+            return
+        self._invocation_count += 1
+        n = self._invocation_count
+        try:
+            os.makedirs(self.debug_output_dir, exist_ok=True)
+
+            label = ("_" + self._debug_label) if self._debug_label else ""
+
+            def _write(suffix, contents):
+                path = os.path.join(self.debug_output_dir, "claude_%03d%s_%s" % (n, label, suffix))
+                mode = "wb" if isinstance(contents, bytes) else "w"
+                with open(path, mode) as f:
+                    f.write(contents if contents is not None else "")
+
+            _write("prompt.txt", prompt)
+            if system_prompt:
+                _write("system_prompt.txt", system_prompt)
+            _write("stdout.json", ret.stdout)
+            _write("stderr.txt", getattr(ret, "stderr", None))
+            self.logger.log("Wrote AI debug output to %s (invocation %d)" % (
+                self.debug_output_dir, n), level=LogLevel.Info)
+        except OSError as e:
+            self.logger.log("Could not write AI debug output: %s" % e, level=LogLevel.Warning)
+
+    def _parse_result(self, ret):
+        stdout = ret.stdout.decode() if isinstance(ret.stdout, bytes) else ret.stdout
+        try:
+            data = json.loads(stdout)
+        except ValueError:
+            self.logger.log("The AI CLI did not return parseable JSON output.", level=LogLevel.Error)
+            return AIResult(False, stdout)
+
+        success = (ret.returncode == 0) and not data.get("is_error", True)
+        return AIResult(success, data.get("result", ""), data.get("session_id"))
+
+    @logEntryExit
+    def resolve_patch_conflicts(self, moz_yaml_path, commit_message, cwd=None,
+                                library_name=None, job_id=None):
+        # Ask the AI to resolve local-patch conflicts for a library. It works in
+        # the checkout (cwd), updates the .patch files / moz.yaml so they apply,
+        # and writes its verdict to CONFLICT_RESOLUTION_RESULT_FILE. Returns the
+        # parsed {"outcome", "details"} dict, or None if it produced no result.
+        # library_name/job_id only label debug output; library_name defaults to
+        # the moz.yaml's parent directory name.
+        library = library_name or os.path.basename(os.path.dirname(moz_yaml_path)) or "library"
+        self._debug_label = "%s_job%s" % (library, job_id if job_id is not None else "none")
+
+        instructions = load_prompt("conflict_resolution_details",
+                                   moz_yaml_path=moz_yaml_path,
+                                   patch_fix_commit_message=commit_message)
+        prompt = load_prompt("conflict_resolution", conflict_resolution_instructions=instructions)
+        system_prompt = load_prompt("system")
+
+        self._run_prompt(prompt, system_prompt=system_prompt, cwd=cwd)
+
+        result_path = os.path.join(cwd, CONFLICT_RESOLUTION_RESULT_FILE) if cwd else CONFLICT_RESOLUTION_RESULT_FILE
+        try:
+            with open(result_path) as f:
+                resolution = json.load(f)
+            os.remove(result_path)
+        except (OSError, ValueError):
+            self.logger.log("Could not read %s after AI conflict resolution." % CONFLICT_RESOLUTION_RESULT_FILE, level=LogLevel.Warning)
+            return None
+        self.logger.log("AI conflict resolution outcome: %s" % resolution.get("outcome"), level=LogLevel.Info)
+        return resolution

components/bugzilla.py

@@ -152,6 +152,40 @@ def COULD_NOT_GENERAL_ERROR(action, errormessage=None):
         s += "\nUpdatebot will be unable to do anything more for this library version."
         return s
 
+    @staticmethod
+    def COULD_NOT_GENERAL_ERROR_WITH_AI(action, initialerrormessage=None, ai_outcome=None, ai_details=None):
+        s = "Updatebot encountered an error while trying to %s" % action
+        if initialerrormessage:
+            s += " with the following message:\n\n"
+            for line in initialerrormessage.split("\n"):
+                s += "> " + line + "\n"
+        s += "\nAfter getting this error, Updatebot asked its AI assistant to help"
+        if ai_outcome:
+            s += ", which reported an outcome of '%s'." % ai_outcome
+        else:
+            s += ", which seemingly failed with no outcome at all."
+        if ai_details:
+            s += "\n\n"
+            for line in ai_details.split("\n"):
+                s += "> " + line + "\n"
+        s += "\nUpdatebot will be unable to do anything more for this library version."
+        return s
+
+    @staticmethod
+    def AI_RESOLVED_PATCH_CONFLICTS(outcome, details):
+        explanations = {
+            "trivial success": "This means the patches were updated in a straightforward way and should be reliable.",
+            "uncertain success": "This means resolving the conflicts required non-trivial judgement and the result should be reviewed carefully.",
+        }
+        s = "Updatebot's AI assistant resolved conflicts while applying the local patches and reported an outcome of '%s'.\n\n" % outcome
+        explanation = explanations.get(outcome)
+        if explanation:
+            s += explanation + "\n\n"
+        if details:
+            for line in details.split("\n"):
+                s += "> " + line + "\n"
+        return s
+
     @staticmethod
     def COULD_NOT_VENDOR_ALL_FILES(library, errormessage):
         s = "`./mach vendor %s` reported an error editing moz.build files:\n" % library.yaml_path

components/commandprovider.py

@@ -20,6 +20,7 @@ def _update_config(self, additional_config):
         self.infolog = partial(self.logger.log, level=LogLevel.Info)
         self.debuglog = partial(self.logger.log, level=LogLevel.Debug)
 
-    def run(self, args, shell=False, clean_return=True):
+    def run(self, args, shell=False, clean_return=True, cwd=None, stdin_path=None, timeout=60 * 20, env=None):
         return _run(args, shell=shell, clean_return=clean_return,
-                    errorlog=self.errorlog, infolog=self.infolog, debuglog=self.debuglog)
+                    errorlog=self.errorlog, infolog=self.infolog, debuglog=self.debuglog,
+                    cwd=cwd, stdin_path=stdin_path, timeout=timeout, env=env)

components/commandrunner.py

@@ -23,7 +23,8 @@ def do_nothing(*args, **kwargs):
 """
 
 
-def _run(args, shell, clean_return, errorlog=do_nothing, infolog=do_nothing, debuglog=do_nothing):
+def _run(args, shell, clean_return, errorlog=do_nothing, infolog=do_nothing, debuglog=do_nothing,
+         cwd=None, stdin_path=None, timeout=60 * 20, env=None):
     ran_to_completion = False
     stdout = None
     stderr = None
@@ -49,9 +50,15 @@ def _run(args, shell, clean_return, errorlog=do_nothing, infolog=do_nothing, deb
 
     start = time.time()
     infolog("Running", args)
+    # Optionally feed a file to the process's stdin (used to pass large input,
+    # e.g. a Claude prompt, without hitting command-line length limits).
+    stdin_handle = open(stdin_path, "rb") if stdin_path else None
+    # Merge any extra env (e.g. secrets) over the inherited environment.
+    run_env = {**os.environ, **env} if env else None
     try:
         ret = subprocess.run(
-            args, shell=shell, stdout=PIPE, stderr=PIPE, timeout=60 * 20)
+            args, shell=shell, stdout=PIPE, stderr=PIPE, timeout=timeout,
+            cwd=cwd, stdin=stdin_handle, env=run_env)
     except subprocess.TimeoutExpired as e:
         ran_to_completion = False
         stdout = e.stdout
@@ -61,6 +68,9 @@ def _run(args, shell, clean_return, errorlog=do_nothing, infolog=do_nothing, deb
         ran_to_completion = True
         stdout = ret.stdout.decode()
         stderr = ret.stderr.decode()
+    finally:
+        if stdin_handle:
+            stdin_handle.close()
 
     if not ran_to_completion:
         errorlog("Command Timed Out. Will abort....")