Unify token estimation interface in optimiser

src/Contracts/LLMTokenOptimisationProxyInterface.php

@@ -19,18 +19,31 @@
 /**
  * Contract for proxies that reduce the number of tokens a payload will consume.
  *
- * Tool and command output is often the largest, noisiest contributor to an agent's context.
- * An optimisation proxy compresses that text before it reaches the model — stripping padding,
- * deduplicating, and summarising — so the same information costs fewer tokens.
+ * Command output is often the largest, noisiest contributor to an agent's context. An
+ * optimisation proxy compresses that text before it reaches the model (stripping padding,
+ * deduplicating, summarising) so the same information costs fewer tokens.
+ *
+ * **Optimisation is lossy by design.** Implementations rewrite the text and may drop detail the
+ * caller considers incidental, such as file names in a directory listing. Only route text through
+ * a proxy when the model needs to *read* it. Never route content the model must reproduce
+ * verbatim (source files, whole-file edit payloads, anything round-tripped back to disk): at best
+ * the proxy returns it unchanged for no gain, at worst it corrupts it. Text that is already
+ * structured or summarised at the source has nothing left to squeeze either.
  *
  * The reference implementation is an RTK adapter (https://github.com/rtk-ai/rtk), which shells
  * out to the `rtk` binary; other strategies (tokenizer-based pruning, summarisation) can
  * implement the same contract.
+ *
+ * Nothing in papi wires an optimiser in automatically. Agent middleware only sees the request
+ * prompt (tool results are resolved inside the run loop), so the integration point is your own
+ * tool handler: run the command, pipe its output through the proxy, return the compressed text.
  */
-interface LLMTokenOptimisationProxyInterface
+interface LLMTokenOptimisationProxyInterface extends TokenEstimatorInterface
 {
     /**
-     * Optimise a block of text (e.g. captured tool output) before it enters the context.
+     * Optimise a block of text (e.g. captured command output) before it enters the context.
+     *
+     * Lossy: see the class docblock for what is safe to pass.
      *
      * @param string $content The raw text to compress
      * @param array{
@@ -46,27 +59,18 @@ public function optimise(string $content, array $options = []): OptimisationResu
      * Run a command through the proxy and return its optimised output.
      *
      * Intended for read-only developer commands (git, grep, ls, test runners) whose verbose
-     * output is the real token cost. Implementations may execute the command to measure the
-     * saving, so only pass side-effect-free commands.
+     * output is the real token cost. Implementations measure the saving by also running the
+     * command unoptimised, so pass only side-effect-free commands, and pass `measure: false` to
+     * skip that second execution when the saving figure is not worth the latency (the result's
+     * `tokensBefore` is then null).
      *
      * @param string $command The command line to run (e.g. "git status")
      * @param array{
+     *     measure?: bool,
      *     ultraCompact?: bool,
      * } $options Strategy-specific options
      *
      * @return OptimisationResult The optimised output plus before/after token estimates
      */
     public function optimiseCommand(string $command, array $options = []): OptimisationResult;
-
-    /**
-     * Estimate the number of tokens a block of text would consume.
-     *
-     * A cheap heuristic used to decide whether a payload is large enough to be worth optimising;
-     * it does not call a real tokenizer.
-     *
-     * @param string $content The text to measure
-     *
-     * @return int The estimated token count
-     */
-    public function estimateTokens(string $content): int;
 }

src/Contracts/TokenEstimatorInterface.php

@@ -0,0 +1,38 @@
+<?php
+
+/*
+ * This file is part of PapiAI,
+ * A simple but powerful PHP library for building AI agents.
+ *
+ * (c) Marcello Duarte <marcello.duarte@gmail.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+declare(strict_types=1);
+
+namespace PapiAI\Core\Contracts;
+
+/**
+ * Contract for anything that can put a token figure on a block of text.
+ *
+ * Deliberately a single method so components that only need to size a payload (ingestion budgets,
+ * context windows, "is this worth compressing?" checks) depend on nothing more than that.
+ * Richer contracts extend it: {@see LLMTokenOptimisationProxyInterface} is an estimator that can
+ * also compress.
+ *
+ * Implementations may be heuristic or exact. {@see \PapiAI\Core\HeuristicTokenEstimator} is the
+ * zero-dependency default; a real tokenizer satisfies the same contract with better numbers.
+ */
+interface TokenEstimatorInterface
+{
+    /**
+     * Estimate the number of tokens a block of text would consume.
+     *
+     * @param string $content The text to measure
+     *
+     * @return int The estimated token count (never negative)
+     */
+    public function estimateTokens(string $content): int;
+}

src/HeuristicTokenEstimator.php

@@ -0,0 +1,46 @@
+<?php
+
+/*
+ * This file is part of PapiAI,
+ * A simple but powerful PHP library for building AI agents.
+ *
+ * (c) Marcello Duarte <marcello.duarte@gmail.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+declare(strict_types=1);
+
+namespace PapiAI\Core;
+
+use PapiAI\Core\Contracts\TokenEstimatorInterface;
+
+/**
+ * Zero-dependency token estimator using the familiar bytes-per-token rule of thumb.
+ *
+ * The unit is **bytes**, not characters: `strlen()` divided by a fixed ratio (4 by default,
+ * roughly right for English prose and source code under the common BPE vocabularies). That choice
+ * is deliberate rather than incidental. A UTF-8 character can span several bytes, so multibyte
+ * text estimates high, and erring high is the safe direction for anything spending a budget: a
+ * caller under-fills the context window instead of blowing past it.
+ *
+ * Naming the unit once, here, also keeps every estimate in the ecosystem comparable. Use it
+ * wherever an approximate figure is enough (ingestion budgets, deciding whether a payload is worth
+ * compressing) and swap in a real tokenizer when exact counts matter.
+ */
+final class HeuristicTokenEstimator implements TokenEstimatorInterface
+{
+    /**
+     * @param positive-int $bytesPerToken Average bytes each token is assumed to consume
+     */
+    public function __construct(
+        private readonly int $bytesPerToken = 4,
+    ) {
+    }
+
+    public function estimateTokens(string $content): int
+    {
+        return (int) ceil(strlen($content) / $this->bytesPerToken);
+    }
+}

src/OptimisationResult.php

@@ -20,44 +20,65 @@
  * Holds the optimised text alongside the estimated token counts before and after, so callers
  * can decide whether the saving was worthwhile and report on it. Token counts are estimates,
  * not the output of a real tokenizer.
+ *
+ * The baseline can be unknown: measuring it means processing (or running) the content twice, so
+ * a caller may opt out. `$tokensBefore` is then null and the saving is unreportable rather than
+ * reported as zero.
  */
 final class OptimisationResult
 {
     /**
-     * @param string $optimised    The optimised (compressed) text
-     * @param int    $tokensBefore Estimated tokens in the original content
-     * @param int    $tokensAfter  Estimated tokens in the optimised content
-     * @param string $strategy     Identifier of the strategy used (e.g. "rtk:pipe", "rtk:command")
+     * @param string   $optimised    The optimised (compressed) text
+     * @param int|null $tokensBefore Estimated tokens in the original content, or null if unmeasured
+     * @param int      $tokensAfter  Estimated tokens in the optimised content
+     * @param string   $strategy     Identifier of the strategy used (e.g. "rtk:pipe", "rtk:command")
      */
     public function __construct(
         public readonly string $optimised,
-        public readonly int $tokensBefore,
+        public readonly ?int $tokensBefore,
         public readonly int $tokensAfter,
         public readonly string $strategy = '',
     ) {
     }
 
+    /**
+     * Whether the original content was measured, and so whether a saving can be reported.
+     */
+    public function isMeasured(): bool
+    {
+        return $this->tokensBefore !== null;
+    }
+
     /**
      * Number of tokens saved (never negative).
      *
-     * @return int The estimated tokens saved
+     * @return int|null The estimated tokens saved, or null when the baseline was not measured
      */
-    public function tokensSaved(): int
+    public function tokensSaved(): ?int
     {
+        if ($this->tokensBefore === null) {
+            return null;
+        }
+
         return max(0, $this->tokensBefore - $this->tokensAfter);
     }
 
     /**
      * Percentage of tokens saved, rounded to one decimal place.
      *
-     * @return float The saving as a percentage, or 0.0 when there was nothing to save
+     * @return float|null The saving as a percentage, 0.0 when there was nothing to save, or null
+     *                    when the baseline was not measured
      */
-    public function savingsPercent(): float
+    public function savingsPercent(): ?float
     {
+        if ($this->tokensBefore === null) {
+            return null;
+        }
+
         if ($this->tokensBefore <= 0) {
             return 0.0;
         }
 
-        return round($this->tokensSaved() / $this->tokensBefore * 100, 1);
+        return round((int) $this->tokensSaved() / $this->tokensBefore * 100, 1);
     }
 }

tests/Unit/HeuristicTokenEstimatorTest.php

@@ -0,0 +1,33 @@
+<?php
+
+declare(strict_types=1);
+
+use PapiAI\Core\Contracts\TokenEstimatorInterface;
+use PapiAI\Core\HeuristicTokenEstimator;
+
+describe('HeuristicTokenEstimator', function () {
+    it('is a token estimator', function () {
+        expect(new HeuristicTokenEstimator())->toBeInstanceOf(TokenEstimatorInterface::class);
+    });
+
+    it('estimates four bytes per token by default', function () {
+        expect((new HeuristicTokenEstimator())->estimateTokens(str_repeat('a', 400)))->toBe(100);
+    });
+
+    it('rounds partial tokens up', function () {
+        expect((new HeuristicTokenEstimator())->estimateTokens('abcde'))->toBe(2);
+    });
+
+    it('returns zero for empty content', function () {
+        expect((new HeuristicTokenEstimator())->estimateTokens(''))->toBe(0);
+    });
+
+    it('accepts a different bytes-per-token ratio', function () {
+        expect((new HeuristicTokenEstimator(2))->estimateTokens(str_repeat('a', 400)))->toBe(200);
+    });
+
+    it('counts bytes, so multibyte text estimates high', function () {
+        // 3 characters but 9 bytes: a character-based rule of thumb would say 1 token, not 3.
+        expect((new HeuristicTokenEstimator())->estimateTokens('日本語'))->toBe(3);
+    });
+});

tests/Unit/OptimisationResultTest.php

@@ -41,6 +41,17 @@
     it('returns zero percent when there is nothing to measure', function () {
         $result = new OptimisationResult('', 0, 0);
 
+        expect($result->isMeasured())->toBeTrue();
         expect($result->savingsPercent())->toBe(0.0);
     });
+
+    it('reports an unmeasured baseline as unknown rather than zero', function () {
+        $result = new OptimisationResult('compact', null, 40, 'rtk:command');
+
+        expect($result->isMeasured())->toBeFalse();
+        expect($result->tokensBefore)->toBeNull();
+        expect($result->tokensSaved())->toBeNull();
+        expect($result->savingsPercent())->toBeNull();
+        expect($result->tokensAfter)->toBe(40);
+    });
 });