diff --git a/.github/workflows/retrain.yml b/.github/workflows/retrain.yml new file mode 100644 index 0000000..1de3293 --- /dev/null +++ b/.github/workflows/retrain.yml @@ -0,0 +1,88 @@ +name: Retrain detector weights + +# Refits the detector's metric weights from training/corpus.jsonl and opens a +# pull request when the fit improves — the model never updates itself unattended. +# See training/README.md. + +on: + workflow_dispatch: {} + push: + branches: [main, master] + paths: + - 'training/corpus.jsonl' + schedule: + # Weekly, Monday 06:00 UTC — picks up any corpus growth merged during the week. + - cron: '0 6 * * 1' + +# The job needs to push a branch and open a PR. +permissions: + contents: write + pull-requests: write + +concurrency: + group: retrain-weights + cancel-in-progress: true + +jobs: + retrain: + name: Fit + gate + propose + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install package + run: | + python -m pip install --upgrade pip + pip install -e . + + - name: Fit weights (writes only if the held-out gate passes) + id: fit + run: | + python training/train_weights.py --write | tee /tmp/train_report.txt + # Stamp the training date (Date is unavailable inside the trainer). + python - <<'PY' + import json, datetime, pathlib + p = pathlib.Path("texthumanize/detector_weights.json") + d = json.loads(p.read_text()) + if d.get("fitted"): + d["trained_at"] = datetime.date.today().isoformat() + p.write_text(json.dumps(d, indent=2, ensure_ascii=False) + "\n") + PY + + - name: Verify the fitted weights still pass the suite + run: | + pip install -e ".[dev]" || pip install pytest + pytest tests/ -k "detect or golden or snapshot" -q --timeout=120 + + # Open a PR with the gh CLI (pre-installed on the runner) rather than a + # third-party action — nothing to pin, no supply-chain surface. + - name: Open a pull request if the weights changed + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + if git diff --quiet -- texthumanize/detector_weights.json; then + echo "weights unchanged — nothing to propose." + exit 0 + fi + branch="auto/retrain-weights" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B "$branch" + git add texthumanize/detector_weights.json + git commit -m "chore(detector): refit weights from corpus" + git push --force-with-lease origin "$branch" + if gh pr view "$branch" --json state -q .state 2>/dev/null | grep -q OPEN; then + echo "PR already open; branch updated." + else + gh pr create --base "${GITHUB_REF_NAME}" --head "$branch" \ + --title "Retrain: updated detector weights" \ + --body-file /tmp/train_report.txt \ + --label automated --label detector + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 7de9894..ee1aef6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,38 @@ Format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +## [0.35.0] - 2026-07-21 + +### Detection quality (evidence-driven; see the extension's research/ catalogs) +- **New `structure` metric** — scores the enumeration/list-intro/participial-tail/ + negative-parallelism scaffolding of instruction-tuned writing. This is the tell + that survives paraphrase and model-generation changes, and the one the old + metric set missed entirely (assistant-register text scored ~40%). +- **Fixed two inverted signals.** `voice` no longer scores passive as AI: in + English GPT-4o uses agentless passive at ~half the human rate, and in RU/UK/DE + passive+nominal is ordinary bureaucratic register — counting it inverted the + metric on edited human prose. It now leans on nominalization (the robust ×1.5–2 + tell). `punctuation` no longer scores semicolons/colons/em-dashes/« » as AI + (marks of careful human editing; the metric was firing on classic prose). +- **Anti-evasion normalization** — homoglyph and zero-width insertion (which + collapse token-frequency detectors by 42–76 pp in the RAID benchmark) are now + stripped/folded before analysis. Whole-word Cyrillic/Greek text is preserved. +- **Metric weights rebalanced** against measured per-metric separation; weight + moved off metrics that neither separate nor survive paraphrase. +- **AI-cliché dictionary expanded** with the 2025–2026 assistant register + ("Great question", "Let's break down", "Here's the deal", "I hope this helps", + and RU equivalents), feeding both detection and humanization. + +### Media provenance +- **Fixed a false-positive class**: a human image carrying a `Description`/ + `Title`/`Comment`/`Software` text chunk was reported as AI. Generic keywords + now require an actual generation-parameter value; only tool-unique chunk keys + (`parameters`, `workflow`, `sd-metadata`, `invokeai_metadata`, + `sui_image_params`, `dream`) are conclusive on the key name alone. +- **Wider coverage**: ComfyUI `class_type`/`sampler_name`, SwarmUI, and 2026 + generator signatures (Recraft, Seedream/Seedance, Hunyuan, Reve, IPTC + `trainedAlgorithmicMedia`). + ## [0.34.0] - 2026-07-16 ### Security diff --git a/README.md b/README.md index aa3f6a0..5e20f59 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ readability, and internal risk signals; it is not a bypass guarantee. [![Tests](https://img.shields.io/badge/tests-2269%20passed-2ea44f.svg?logo=pytest&logoColor=white)](https://github.com/ksanyok/TextHumanize/actions/workflows/ci.yml)    [![Zero Dependencies](https://img.shields.io/badge/dependencies-zero-brightgreen.svg)]() -[![PyPI](https://img.shields.io/badge/pypi-v0.34.0-3775A9.svg?logo=pypi&logoColor=white)](https://pypi.org/project/texthumanize/) +[![PyPI](https://img.shields.io/badge/pypi-v0.35.0-3775A9.svg?logo=pypi&logoColor=white)](https://pypi.org/project/texthumanize/) [![License](https://img.shields.io/badge/license-Dual%20(Free%20%2B%20Commercial)-blue.svg)](LICENSE)
@@ -195,7 +195,7 @@ git clone https://github.com/ksanyok/TextHumanize.git cd TextHumanize && pip install -e . ``` -> **Tip:** Pin your version for production: `pip install texthumanize==0.34.0` +> **Tip:** Pin your version for production: `pip install texthumanize==0.35.0`
PHP / TypeScript @@ -1475,7 +1475,7 @@ reporting rules, and detector limitations. ``` ┌──────────────────────────────────────────────────────────┐ -│ TextHumanize v0.34.0 — AI Score Benchmark │ +│ TextHumanize v0.35.0 — AI Score Benchmark │ ├──────────────────────────────────────────────────────────┤ │ EN (web/50): 94% → 27% (reduction: -67pp) │ │ EN (web/60): 94% → 23% (reduction: -71pp) │ @@ -1887,7 +1887,18 @@ Try the [Live Demo](https://texthumanize.link/). For local use, the REST API + S --- -## 🆕 What's New in v0.34.0 +## 🆕 What's New in v0.35.0 + +### Detection quality — evidence-driven overhaul (0.35.0) +- **New `structure` metric** — scores the enumeration / list-intro / participial-tail / negative-parallelism *scaffolding* of instruction-tuned writing. This is the tell that survives paraphrasing and model-generation changes, and the one the old metric set missed entirely (chat "assistant register" text used to score ~40% human). +- **Two inverted signals fixed.** `voice` no longer scores passive as AI (English GPT-4o uses agentless passive at ~half the human rate; in RU/UK/DE passive+nominal is ordinary bureaucratic register) — it now leans on nominalization, the robust ×1.5–2 tell. `punctuation` no longer treats semicolons / colons / em-dashes / « » as AI — those are marks of careful human editing, and the metric had been firing on well-edited prose. +- **Anti-evasion normalization** — homoglyph and zero-width insertion (which collapse token-frequency detectors by 42–76 pp in the RAID benchmark) are stripped/folded before analysis; whole-word Cyrillic/Greek text is preserved. +- **Weights rebalanced** against measured per-metric separation, and the AI-cliché dictionary gained the 2025–2026 assistant register — feeding both detection and humanization. +- **Self-improving weights** — a new offline training pipeline (`training/`) fits the metric weights from a labelled corpus and is gated by a benchmark so a release never ships a regression. See [`training/README.md`](training/README.md). + +### Media provenance (0.35.0) +- **Fixed a false-positive class** — a human photo carrying a `Description` / `Title` / `Comment` / `Software` metadata chunk was reported as AI. Generic keys now require an actual generation-parameter value; only tool-unique chunk keys (`parameters`, `workflow`, `sd-metadata`, `invokeai_metadata`, `sui_image_params`, `dream`) are conclusive on the key name alone. +- **Wider coverage** — ComfyUI `class_type` / `sampler_name`, SwarmUI, and 2026 generator signatures (Recraft, Seedream/Seedance, Hunyuan, Reve, IPTC `trainedAlgorithmicMedia`). ### Security & REST API hardening (0.34.0) - **Fixed an unauthenticated reflected SSRF (CWE-918)** in the REST API. `POST /humanize` forwarded a client-supplied `oss_api_url` straight into an outbound request, so an anonymous caller could make the server fetch internal/loopback/cloud-metadata URLs and read the response. Reported responsibly by Natnael Wodsnoen. diff --git a/composer.json b/composer.json index 83ebfaf..3cf5381 100644 --- a/composer.json +++ b/composer.json @@ -1,6 +1,6 @@ { "name": "ksanyok/text-humanize", - "version": "0.34.0", + "version": "0.35.0", "description": "Zero-dependency PHP library for algorithmic text humanization — transforms machine-generated text into natural prose", "type": "library", "keywords": [ diff --git a/js/package-lock.json b/js/package-lock.json index 1c08a52..084a79f 100644 --- a/js/package-lock.json +++ b/js/package-lock.json @@ -1,12 +1,12 @@ { "name": "texthumanize", - "version": "0.34.0", + "version": "0.35.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "texthumanize", - "version": "0.34.0", + "version": "0.35.0", "license": "SEE LICENSE IN LICENSE", "devDependencies": { "@types/node": "^20.0.0", diff --git a/js/package.json b/js/package.json index 09fd85d..e120304 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "texthumanize", - "version": "0.34.0", + "version": "0.35.0", "description": "Algorithmic text humanization — transforms AI-generated text into natural-sounding content", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/js/src/version.ts b/js/src/version.ts index aa7ccda..3475238 100644 --- a/js/src/version.ts +++ b/js/src/version.ts @@ -1 +1 @@ -export const VERSION = '0.34.0'; +export const VERSION = '0.35.0'; diff --git a/package.json b/package.json index 669f842..4bbe9e7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "texthumanize", - "version": "0.34.0", + "version": "0.35.0", "private": true, "description": "Text style normalization & readability engine — Python, TypeScript, PHP", "repository": { diff --git a/php/composer.json b/php/composer.json index 2b71b9d..805a62a 100644 --- a/php/composer.json +++ b/php/composer.json @@ -1,6 +1,6 @@ { "name": "ksanyok/text-humanize", - "version": "0.34.0", + "version": "0.35.0", "description": "Zero-dependency PHP library for algorithmic text humanization — transforms machine-generated text into natural prose", "type": "library", "license": "proprietary", diff --git a/php/src/TextHumanize.php b/php/src/TextHumanize.php index b662566..646caca 100644 --- a/php/src/TextHumanize.php +++ b/php/src/TextHumanize.php @@ -21,7 +21,7 @@ */ class TextHumanize { - public const VERSION = '0.34.0'; + public const VERSION = '0.35.0'; /** * Humanize text — the primary API method. diff --git a/pyproject.toml b/pyproject.toml index f1cdeb8..a25ea70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "texthumanize" -version = "0.34.0" +version = "0.35.0" description = "Algorithmic text humanization with AI detection, tone analysis, paraphrasing, and spinning — 38-stage pipeline, 25 languages, SentenceValidator™, PHANTOM™" readme = "README.md" license = "LicenseRef-Proprietary" @@ -110,7 +110,7 @@ Demo = "https://humanizekit.tester-buyreadysite.website/" include = ["texthumanize*"] [tool.setuptools.package-data] -texthumanize = ["data/*.json", "weights/*.zb85", "_data/*.json.gz"] +texthumanize = ["data/*.json", "weights/*.zb85", "_data/*.json.gz", "detector_weights.json"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/tests/test_detector_weights.py b/tests/test_detector_weights.py new file mode 100644 index 0000000..42c9106 --- /dev/null +++ b/tests/test_detector_weights.py @@ -0,0 +1,50 @@ +"""The fitted-weights loading mechanism (training/ pipeline output).""" +from __future__ import annotations + +import json +from importlib.resources import files + +from texthumanize.detectors import AIDetector + + +def test_shipped_weights_file_is_valid(): + raw = (files("texthumanize") / "detector_weights.json").read_text("utf-8") + data = json.loads(raw) + assert data["schema"] == "texthumanize.detector_weights.v1" + w = data["weights"] + # Every shipped weight is a known metric with a sane, non-negative value. + for metric, value in w.items(): + assert metric in AIDetector._WEIGHTS, f"unknown metric {metric}" + assert isinstance(value, (int, float)) and value >= 0 + assert sum(w.values()) > 0 + + +def test_resolved_weights_cover_all_metrics(): + resolved = AIDetector._load_fitted_weights() + # Merge keeps every default metric even if the file omits some. + assert set(resolved) == set(AIDetector._WEIGHTS) + assert all(v >= 0 for v in resolved.values()) + + +def test_invalid_weights_fall_back_to_defaults(monkeypatch): + # A broken payload must not throw and must yield the hand-tuned defaults. + def boom(*a, **k): + raise ValueError("corrupt") + monkeypatch.setattr("texthumanize.detectors.json.loads", boom) + fell_back = AIDetector._load_fitted_weights() + assert fell_back == dict(AIDetector._WEIGHTS) + + +def test_detection_unchanged_by_bootstrap_weights(): + # The shipped bootstrap equals the hand-tuned weights, so a detection with + # the file present must match one forced onto the raw defaults. + text = ("In today's rapidly evolving digital landscape, it is important to " + "note that leveraging synergistic solutions can significantly enhance " + "productivity. Furthermore, organizations must carefully consider the " + "multifaceted implications of these transformative technologies.") + AIDetector._fitted_weights_cache = None + with_file = AIDetector().detect(text, "en").ai_probability + AIDetector._fitted_weights_cache = dict(AIDetector._WEIGHTS) + forced = AIDetector().detect(text, "en").ai_probability + AIDetector._fitted_weights_cache = None + assert abs(with_file - forced) < 1e-9 diff --git a/tests/test_golden.py b/tests/test_golden.py index 50e2064..0066558 100644 --- a/tests/test_golden.py +++ b/tests/test_golden.py @@ -321,7 +321,17 @@ class TestFrozenSnapshots: "is being actively pursued." ) - EXPECTED_EN_HASH = "c07a5f8d19d78cbaec35465f04190435" + # Обновлён после добавления словаря ИИ-клише (lang/_ai_cliches.py) и + # перехода в decancel на «сначала совпадение, потом монетка». + # + # ВНИМАНИЕ: этот снапшот фиксирует, что вывод стабилен, а НЕ что он + # хорош. На этом входе Python-конвейер до сих пор стакает вставки — + # «Here's the deal: and,», «Now, actively, plus,», «(and this is key)» — + # потому что несколько независимых этапов вставляют разговорный + # маркер в одно и то же начало предложения, не зная друг о друге. + # Так было и до этой правки. JS-порт в расширении такого не делает; + # библиотеке нужна та же работа по схлопыванию наложенных вставок. + EXPECTED_EN_HASH = "7eb50b5da1f66edd9acb0d21a304920c" def _md5(self, text: str) -> str: return hashlib.md5(text.encode()).hexdigest() diff --git a/texthumanize/__init__.py b/texthumanize/__init__.py index 1a20b97..3af7f82 100644 --- a/texthumanize/__init__.py +++ b/texthumanize/__init__.py @@ -40,7 +40,7 @@ import types as _types from typing import Any -__version__ = "0.34.0" +__version__ = "0.35.0" __author__ = "TextHumanize Contributors" __license__ = "Personal Use Only" diff --git a/texthumanize/decancel.py b/texthumanize/decancel.py index d5bda6a..2354b64 100644 --- a/texthumanize/decancel.py +++ b/texthumanize/decancel.py @@ -273,10 +273,17 @@ def _replace_phrases(self, text: str, prob: float) -> str: if self._changes_made >= self._max_changes: break - if not coin_flip(prob, self.rng): + # Сначала ищем совпадение, только потом бросаем монетку: иначе + # каждая фраза словаря тратит случайное число независимо от того, + # есть ли она в тексте, и пополнение словарей сдвигает весь поток + # RNG — из-за чего менялись замены в местах, которых правка + # вообще не касалась. + matches = list(pattern.finditer(text)) + if not matches: continue - matches = list(pattern.finditer(text)) + if not coin_flip(prob, self.rng): + continue for match in matches: if has_placeholder(text[max(0, match.start()-5):match.end()+5]): @@ -325,10 +332,13 @@ def _replace_words(self, text: str, prob: float) -> str: if self._changes_made >= self._max_changes: break - if not coin_flip(prob, self.rng): + # Как и в _replace_phrases: сперва совпадение, потом монетка. + matches = list(pattern.finditer(text)) + if not matches: continue - matches = list(pattern.finditer(text)) + if not coin_flip(prob, self.rng): + continue for match in reversed(matches): # Обратный порядок, чтобы не сбить индексы if self._changes_made >= self._max_changes: diff --git a/texthumanize/detector_weights.json b/texthumanize/detector_weights.json new file mode 100644 index 0000000..1c88784 --- /dev/null +++ b/texthumanize/detector_weights.json @@ -0,0 +1,32 @@ +{ + "schema": "texthumanize.detector_weights.v1", + "fitted": false, + "note": "Bootstrap = current hand-tuned weights. The training pipeline (training/) refits these from a labelled corpus and opens a PR when it beats the benchmark. Values are the per-metric contribution to P(AI); non-negative, need not sum to 1 (normalized at use).", + "trained_at": null, + "corpus_size": null, + "metrics": { + "accuracy": null, + "auroc": null + }, + "weights": { + "pattern": 0.18, + "burstiness": 0.13, + "voice": 0.1, + "stylometry": 0.1, + "entity": 0.08, + "structure": 0.08, + "discourse": 0.06, + "rhythm": 0.05, + "entropy": 0.04, + "opening": 0.03, + "grammar": 0.03, + "vocabulary": 0.02, + "perplexity": 0.02, + "semantic_rep": 0.02, + "topic_sentence": 0.02, + "coherence": 0.02, + "readability": 0.005, + "punctuation": 0.005, + "zipf": 0.01 + } +} diff --git a/texthumanize/detectors.py b/texthumanize/detectors.py index 9596f92..2aa1da6 100644 --- a/texthumanize/detectors.py +++ b/texthumanize/detectors.py @@ -22,6 +22,7 @@ from __future__ import annotations +import json import logging import math import re @@ -68,6 +69,7 @@ class DetectionResult: entity_score: float = 0.0 # Специфичность упоминаний voice_score: float = 0.0 # Passive vs active voice topic_sent_score: float = 0.0 # Topic sentence паттерн + structure_score: float = 0.0 # Enumeration/scaffold structure (2026) # Домен и адаптация detected_domain: str = "general" # academic/news/blog/legal/social/code_docs/general @@ -145,6 +147,58 @@ def _get_ai_words() -> dict[str, dict[str, set[str]]]: _AI_WORDS = _load_ai_words() return _AI_WORDS +# ═══════════════════════════════════════════════════════════════ +# Anti-evasion input normalization +# ═══════════════════════════════════════════════════════════════ +# Homoglyph and zero-width insertion break token-frequency detectors badly +# (−42…−76 pp in the RAID benchmark), so normalize BEFORE any metric runs. + +_INVISIBLES_RE = re.compile( + "[\u00ad\u200b-\u200f\u202a-\u202e\u2060-\u2064\u2066-\u206f\ufeff]") + +# Confusable Cyrillic/Greek letters → Latin look-alike. Applied ONLY to a word +# that is majority-Latin, so genuine Cyrillic/Greek words are left untouched. +_CONFUSABLE_TO_LATIN = { + "а": "a", "е": "e", "о": "o", "р": "p", "с": "c", "у": "y", "х": "x", + "і": "i", "ј": "j", "ѕ": "s", "һ": "h", "к": "k", "м": "m", "т": "t", + "в": "b", "н": "h", "ё": "e", + "А": "A", "Е": "E", "О": "O", "Р": "P", "С": "C", "У": "Y", "Х": "X", + "І": "I", "В": "B", "Н": "H", "К": "K", "М": "M", "Т": "T", "Ѕ": "S", "Ј": "J", + "α": "a", "ο": "o", "ρ": "p", "ε": "e", "ν": "v", "κ": "k", + "Α": "A", "Β": "B", "Ε": "E", "Ζ": "Z", "Η": "H", "Ι": "I", "Κ": "K", + "Μ": "M", "Ν": "N", "Ο": "O", "Ρ": "P", "Τ": "T", "Υ": "Y", "Χ": "X", +} +_LATIN_CHAR_RE = re.compile(r"[a-z]", re.IGNORECASE) +_CONFUSABLE_BLOCK_RE = re.compile(r"[Ѐ-ӿͰ-Ͽ]") + + +def _fold_word(word: str) -> str: + if not _LATIN_CHAR_RE.search(word): + return word # no Latin → genuine other-script word + latin = confusable = 0 + for ch in word: + if ch.isascii() and ch.isalpha(): + latin += 1 + elif ch in _CONFUSABLE_TO_LATIN: + confusable += 1 + if not confusable or latin < confusable: + return word + return "".join(_CONFUSABLE_TO_LATIN.get(ch, ch) for ch in word) + + +def _normalize_for_detection(text: str) -> str: + """Strip invisibles and fold homoglyphs in Latin-majority words so a + homoglyph/zero-width evasion cannot hide AI markers from token metrics. + Whole-word Cyrillic/Greek text is preserved. See research/text-signals-2026.md. + """ + if not text: + return text + text = _INVISIBLES_RE.sub("", text) + if not _CONFUSABLE_BLOCK_RE.search(text): + return text # fast path: no mixed-script risk + return re.sub(r"\S+", lambda m: _fold_word(m.group(0)), text) + + # ═══════════════════════════════════════════════════════════════ # ОСНОВНОЙ ДЕТЕКТОР # ═══════════════════════════════════════════════════════════════ @@ -157,25 +211,31 @@ class AIDetector: """ # Веса метрик (калиброваны для максимальной точности) + # Rebalanced 2026 against measured per-metric separation (see + # research/text-signals-2026.md): weight moved off metrics that neither + # separate nor survive paraphrase onto the ones that measurably do, plus the + # new `structure` scaffold signal. punctuation/opening de-weighted after the + # inversion fixes; passive dropped from `voice`. _WEIGHTS = { - "pattern": 0.20, # AI patterns — самый сильный сигнал - "burstiness": 0.14, # Сильный сигнал: AI = равномерные предложения - "stylometry": 0.09, # Хорошая дискриминация (0.65 vs 0.19) - "voice": 0.08, # Отличная дискриминация (0.76 vs 0.00) - "entity": 0.07, # Хорошая (0.69 vs 0.17) - "opening": 0.06, - "grammar": 0.05, + "pattern": 0.18, # AI patterns — самый сильный сигнал + "burstiness": 0.13, # AI = равномерные предложения + "voice": 0.10, # номинализация (пассив убран) + "stylometry": 0.10, # хорошая дискриминация, парафраз-устойчива + "entity": 0.08, + "structure": 0.08, # перечисления/каркас (2026) + "discourse": 0.06, # дискурсивная структура + "rhythm": 0.05, "entropy": 0.04, - "discourse": 0.04, # Дискурсивная структура - "vocabulary": 0.04, - "rhythm": 0.04, - "perplexity": 0.03, - "semantic_rep": 0.03, # Семантические повторы + "opening": 0.03, + "grammar": 0.03, + "vocabulary": 0.02, + "perplexity": 0.02, + "semantic_rep": 0.02, "topic_sentence": 0.02, - "readability": 0.02, # Слабая дискриминация (~0.50 для всех) - "punctuation": 0.02, # Слабая дискриминация - "coherence": 0.02, # Слабая дискриминация - "zipf": 0.01, # Ненадёжен для коротких текстов + "coherence": 0.02, + "readability": 0.005, # почти нейтральна + "punctuation": 0.005, # де-инвертирована, слабый сигнал + "zipf": 0.01, } # Domain-specific weight adjustments @@ -278,9 +338,44 @@ def _detect_domain(text: str, words: list[str]) -> str: return "general" + # Fitted weights are loaded once from detector_weights.json (produced by the + # offline training pipeline in training/) and cached. When the file is + # absent or invalid, the hand-tuned _WEIGHTS above are used. This is the + # library's "learned model": a transparent, inspectable set of numbers, + # trained offline and versioned in git — no runtime ML, no black box. + _fitted_weights_cache: dict[str, float] | None = None + + @classmethod + def _load_fitted_weights(cls) -> dict[str, float]: + """Load + validate detector_weights.json, falling back to _WEIGHTS.""" + try: + from importlib.resources import files + raw = (files("texthumanize") / "detector_weights.json").read_text("utf-8") + data = json.loads(raw) + fitted = data.get("weights", data) + if not isinstance(fitted, dict): + return dict(cls._WEIGHTS) + merged = dict(cls._WEIGHTS) + for metric, value in fitted.items(): + # Only override known metrics with sane, non-negative numbers. + if metric in merged and isinstance(value, (int, float)) and value >= 0: + merged[metric] = float(value) + total = sum(merged.values()) + if total <= 0: + return dict(cls._WEIGHTS) + return merged + except Exception: + return dict(cls._WEIGHTS) + + @classmethod + def _resolved_weights(cls) -> dict[str, float]: + if cls._fitted_weights_cache is None: + cls._fitted_weights_cache = cls._load_fitted_weights() + return cls._fitted_weights_cache + def _get_adaptive_weights(self, domain: str) -> dict[str, float]: """Get domain-adjusted metric weights.""" - weights = dict(self._WEIGHTS) + weights = dict(self._resolved_weights()) mods = self._DOMAIN_WEIGHT_MODS.get(domain, {}) for metric, delta in mods.items(): if metric in weights: @@ -304,6 +399,9 @@ def detect(self, text: str, lang: str | None = None) -> DetectionResult: Returns: DetectionResult с подробными метриками. """ + if isinstance(text, str): + text = _normalize_for_detection(text) + effective_lang = lang or self.lang if effective_lang == "auto": from texthumanize.lang_detect import detect_language @@ -349,6 +447,7 @@ def detect(self, text: str, lang: str | None = None) -> DetectionResult: result.entity_score = self._calc_entity_specificity(text, words) result.voice_score = self._calc_voice(text, sentences) result.topic_sent_score = self._calc_topic_sentence(text, sentences) + result.structure_score = self._calc_structure(text, sentences) # ── Domain detection & adaptive weights ── detected_domain = self._detect_domain(text, words) @@ -375,6 +474,7 @@ def detect(self, text: str, lang: str | None = None) -> DetectionResult: "entity": result.entity_score, "voice": result.voice_score, "topic_sentence": result.topic_sent_score, + "structure": result.structure_score, } # ── Ensemble boosting aggregation ── @@ -1250,44 +1350,20 @@ def _calc_punctuation(self, text: str, sentences: list[str]) -> float: # Per 1000 chars k = 1000 / total_chars - semi_rate = semicolons * k - colon_rate = colons * k - dash_rate = em_dashes * k ellipsis_rate = ellipsis * k excl_rate = exclamations * k - # AI: высокая частота ; и : , низкая ... и ! - # Human: больше ... и !, меньше ; - - # Semicolons: AI ~2-5 per 1K, Human ~0-1 per 1K - semi_score = min(semi_rate / 3.0, 1.0) - - # Colons: AI ~2-4 per 1K, Human ~0.5-1.5 per 1K - colon_score = min(colon_rate / 3.0, 1.0) - - # Em dashes: AI использует идеальные — , human чаще - - dash_score = min(dash_rate / 4.0, 1.0) - - # Ellipsis: human ~1-3 per 1K, AI ~0 - ellipsis_score = max(0, 1.0 - ellipsis_rate / 2.0) - - # Exclamations: human uses more - excl_score = max(0, 1.0 - excl_rate / 2.0) - - # Punctuation diversity: AI uses fewer types - punct_types = sum(1 for v in [semicolons, colons, em_dashes, ellipsis, - exclamations, questions, parens] if v > 0) - diversity_score = max(0, 1.0 - punct_types / 5.0) - - score = ( - semi_score * 0.2 - + colon_score * 0.15 - + dash_score * 0.15 - + ellipsis_score * 0.15 - + excl_score * 0.1 - + diversity_score * 0.25 - ) - return max(0.0, min(1.0, score)) + # Semicolons, colons, em dashes and « » are marks of careful HUMAN + # editing, not AI — scoring them as AI (and penalising punctuation + # diversity) made this metric fire on well-edited prose, i.e. it ran + # inverted (measured human 0.56 > ai 0.48). Em-dash frequency is also + # epoch-dependent and now user-tunable, so it is an unreliable, + # high-false-positive signal. Keep only the weak, low-FP direction: + # AI assistant/blog prose rarely uses exclamations or trailing ellipses. + del semicolons, colons, em_dashes, questions, parens + calm_score = max(0.0, 1.0 - (excl_rate + ellipsis_rate) / 2.0) + # Blend gently toward neutral so this never dominates or inverts again. + return max(0.0, min(1.0, 0.42 + calm_score * 0.16)) # ─── 8. КОГЕРЕНТНОСТЬ ───────────────────────────────────── @@ -2122,9 +2198,70 @@ def _calc_voice(self, text: str, sentences: list[str]) -> float: active_ratio = active_markers / total_clauses if total_clauses > 0 else 0 active_score = max(0.0, 1.0 - active_ratio / 0.3) - score = passive_score * 0.35 + nom_score * 0.35 + active_score * 0.30 + # Nominalization (heavy noun style) is the robust AI tell — ×1.5–2 in + # instruction-tuned output (Reinhart, PNAS 2025). The PASSIVE ratio is + # deliberately NOT scored as AI: in English GPT-4o uses agentless passive + # at ~half the human rate (high passive skews HUMAN), and in RU/UK/DE + # passive+nominal is ordinary bureaucratic register. Counting it inverted + # the metric on edited human prose. `passive_score` kept for reporting. + _ = passive_score + score = nom_score * 0.62 + active_score * 0.38 return max(0.0, min(1.0, score)) + # ─── 19. СТРУКТУРНЫЙ КАРКАС (2026) ───────────────────────── + + # Sequential enumeration openers ("First,/Finally", "во-первых", …). + _ENUM_MARKERS = [ + r'(?:^|[.!?…]["»\']?\s+)(?:first|second|third|fourth|fifth|firstly|secondly|thirdly|finally|lastly|next|then|moreover|furthermore)\s*,', + r'(?:^|[.!?…]\s+)(?:во-первых|во-вторых|в-третьих|в-четвёртых|наконец|далее|затем)\b', + r'(?:^|[.!?…]\s+)(?:по-перше|по-друге|по-третє|нарешті|далі|потім)\b', + r'(?:^|[.!?…]\s+)(?:erstens|zweitens|drittens|schließlich|außerdem)\b', + r'(?:^|[.!?…]\s+)(?:primero|segundo|tercero|finalmente|por último|además)\b', + r'(?:^|\n)\s*(?:\d{1,2}[.)]\s|[-*•]\s)', + ] + _LIST_INTRO = [ + r'\b(?:following|these|key|main|several|important|below|steps|components|reasons|benefits|factors|ways|points|aspects|elements)\s*:', + r'\b(?:следующ\w+|ключев\w+|основн\w+|несколько|важн\w+|причин\w+|преимуществ\w+|факторов|шаг\w+|компонент\w+)\s*:', + ] + _PARTICIPIAL_TAILS = [ + r',\s+(?:enabling|providing|allowing|ensuring|highlighting|making|creating|offering|leveraging|fostering|driving|improving|reducing|increasing|helping|leading|resulting|reflecting|showcasing|emphasizing|underscoring|empowering|streamlining|enhancing|delivering)\b', + ] + _NEG_PARALLEL = [ + r'\bnot only\b[^.!?]{3,60}\bbut also\b', + r"\bit'?s not (?:just |merely |simply )?[^.!?,]{2,40},?\s+it'?s\b", + r'\bне только\b[^.!?]{3,60}\bно и\b', + r'\bне просто\b[^.!?]{2,40},?\s+а\b', + ] + + def _calc_structure(self, text: str, sentences: list[str]) -> float: + """Структурный каркас instruction-tuned письма: перечисления, + list-intro двоеточия, причастные -ing хвосты, негативный параллелизм. + + Этот сигнал переживает парафраз и смену поколений моделей (в отличие от + лексики) и был полностью пропущен старым набором метрик — ассистентский + регистр набирал ~40%. См. research/text-signals-2026.md. + + Возвращает: 0.0 (нет каркаса = human) — 1.0 (жёсткий каркас = AI) + """ + n_sent = max(1, len(sentences)) + low = text.lower() + + def hits(pats: list[str]) -> int: + return sum(len(re.findall(p, low, re.IGNORECASE | re.MULTILINE)) for p in pats) + + enum_hits = hits(self._ENUM_MARKERS) + list_hits = hits(self._LIST_INTRO) + part_hits = hits(self._PARTICIPIAL_TAILS) + neg_hits = hits(self._NEG_PARALLEL) + + enum_score = min(enum_hits / n_sent / 0.5, 1.0) + list_score = min(list_hits / n_sent / 0.25, 1.0) + part_score = min(part_hits / n_sent / 0.4, 1.0) + neg_score = min(neg_hits / n_sent / 0.2, 1.0) + + raw = enum_score * 0.42 + part_score * 0.28 + list_score * 0.18 + neg_score * 0.12 + return max(0.0, min(1.0, 0.34 + raw * 0.62)) + # ─── 18. TOPIC SENTENCE ПАТТЕРН ─────────────────────────── def _calc_topic_sentence(self, text: str, sentences: list[str]) -> float: @@ -2225,8 +2362,8 @@ def _ensemble_aggregate( # Если ключевые «сильные» метрики все высокие/низкие — # это сильный сигнал независимо от остальных strong_metrics = [ - "pattern", "burstiness", "opening", "stylometry", - "discourse", "voice", "grammar", + "pattern", "burstiness", "stylometry", + "discourse", "voice", "structure", ] strong_vals = [scores.get(m, 0.5) for m in strong_metrics] strong_avg = statistics.mean(strong_vals) diff --git a/texthumanize/lang/__init__.py b/texthumanize/lang/__init__.py index eed516d..44730da 100644 --- a/texthumanize/lang/__init__.py +++ b/texthumanize/lang/__init__.py @@ -6,6 +6,7 @@ и любые другие языки через универсальный процессор. """ +from texthumanize.lang import _ai_cliches from texthumanize.lang.ar import LANG_AR from texthumanize.lang.cs import LANG_CS from texthumanize.lang.da import LANG_DA @@ -60,6 +61,11 @@ "vi": LANG_VI, } +# Клише ИИ-текстов детектор штрафует, но в словарях пакетов их не было — +# подмешиваем, чтобы гуманизатор умел их убирать (см. _ai_cliches). +for _pack in LANGUAGES.values(): + _ai_cliches.merge_into(_pack) + _LANG_PACK_CACHE: dict[str, dict] = {} # ── Language tiers ───────────────────────────────────────── diff --git a/texthumanize/lang/_ai_cliches.py b/texthumanize/lang/_ai_cliches.py new file mode 100644 index 0000000..e8143b1 --- /dev/null +++ b/texthumanize/lang/_ai_cliches.py @@ -0,0 +1,337 @@ +"""Клише, характерные для текстов, написанных ИИ. + +Детектор уже штрафует эти обороты (см. ``HEDGING_PATTERNS`` в +``detector``), но в словарях языковых пакетов их не было — то есть +гуманизатор не умел их убирать. Из-за этого самые узнаваемые ИИ-зачины +(«In today's rapidly evolving digital landscape», «В современном мире…») +переживали обработку, и оценка почти не менялась. + +Здесь собраны именно *зачины и связки-штампы*, а не канцелярит: канцелярит +живёт в ``bureaucratic`` / ``bureaucratic_phrases`` каждого пакета. Записи +отсюда подмешиваются в ``bureaucratic_phrases`` при сборке ``LANGUAGES`` +(см. ``lang/__init__.py``), поэтому их подхватывают и Python, и экспорт в +JS/PHP-порты без отдельной логики. + +Формат: ``{фраза: [варианты замены]}``. Пустая строка среди вариантов +означает «допустимо просто удалить» — для зачинов это чаще всего самая +человечная правка. Ключи в нижнем регистре; регистр восстанавливается при +замене. + +ВАЖНО про флективные языки (ru, uk, pl, de). Замена обязана сохранять +управление: следующее за фразой слово остаётся в исходном падеже, потому +что мы его не трогаем. Поэтому + + "стал неотъемлемой частью" → "прочно вошёл в" + +недопустимо: «частью» требует родительного («частью нашей жизни»), а +«вошёл в» — винительного, и получается «вошёл в нашей жизни». По той же +причине «играет ключевую роль в» нельзя менять на «определяет» +(предложный → винительный) и «широкий спектр» на «разные» (родительный → +именительный). Если подходящей замены с тем же управлением нет, лучше +оставить один вариант, чем добавить грамматически ломающий. +""" + +from __future__ import annotations + +# ── Английский ────────────────────────────────────────────── +AI_CLICHES_EN: dict[str, list[str]] = { + # Зачины «в наше время» + "in today's rapidly evolving digital landscape": ["these days", "right now", ""], + "in today's rapidly evolving world": ["these days", "right now", ""], + "in today's fast-paced world": ["these days", "right now", ""], + "in today's digital landscape": ["these days", "online today", ""], + "in today's digital age": ["these days", "now", ""], + "in today's world": ["these days", "now", ""], + "in today's society": ["these days", "now", ""], + "in the modern world": ["these days", "now", ""], + "in an increasingly digital world": ["as more moves online", "these days", ""], + "in the ever-evolving landscape of": ["in", "across"], + "in the ever-changing world of": ["in", "across"], + "in the realm of": ["in", "when it comes to"], + "in the world of": ["in", "when it comes to"], + "in the field of": ["in"], + # Хеджирование + "it is important to note that": ["note that", "importantly,", ""], + "it is worth noting that": ["worth noting,", "note that", ""], + "it is worth mentioning that": ["worth mentioning,", ""], + "it is essential to understand that": ["understand that", ""], + "it is crucial to recognize that": ["recognise that", ""], + "it should be emphasized that": ["notably,", ""], + "one must consider": ["consider", "think about"], + "it is undeniable that": ["clearly,", ""], + # Штампы-усилители + "plays a crucial role in": ["matters for", "drives", "is central to"], + "plays a vital role in": ["matters for", "drives", "is central to"], + "plays a pivotal role in": ["matters for", "drives", "is central to"], + "plays a significant role in": ["matters for", "shapes"], + "has become an integral part of": ["is now part of", "is now built into"], + "is an integral part of": ["is part of", "is built into"], + "a testament to": ["a sign of", "proof of"], + "stands as a testament to": ["shows", "proves"], + "serves as a reminder": ["is a reminder", "reminds us"], + "a wide range of": ["many", "plenty of", "all sorts of"], + "a wide variety of": ["many", "plenty of", "all sorts of"], + "a myriad of": ["many", "countless"], + "a plethora of": ["plenty of", "lots of"], + "an array of": ["a set of", "several"], + "navigating the complexities of": ["working through", "dealing with"], + "navigate the complexities of": ["work through", "deal with"], + "delve deeper into": ["look closer at", "dig into"], + "delve into": ["look at", "dig into"], + "shed light on": ["clarify", "explain", "show"], + "pave the way for": ["open the door to", "lead to", "make room for"], + "at the forefront of": ["leading", "ahead in"], + "unlock the potential of": ["get more out of", "make the most of"], + "unlock unprecedented opportunities": ["open up new options", "create new openings"], + "harness the power of": ["use", "put to work"], + "revolutionize the way": ["change how", "reshape how"], + "transform the way": ["change how", "reshape how"], + "striking the right balance": ["getting the balance right", "finding a balance"], + "in an era where": ["now that", "when"], + "the rise of": ["the spread of", "the growth of"], + # Концовки + "in conclusion,": ["so,", "all told,", ""], + "to sum up,": ["in short,", ""], + "in summary,": ["in short,", ""], + "ultimately,": ["in the end,", ""], + "all in all,": ["overall,", ""], + "at the end of the day,": ["in the end,", ""], + # Ассистентский регистр (доминирует у чат-моделей 2025-2026) + "great question": ["", "good point"], + "great question!": [""], + "let's dive in": ["", "here goes"], + "let's dive into": ["let's look at"], + "let's break it down": [""], + "let's break down": ["here's"], + "let's explore": ["look at"], + "here's the thing": [""], + "here's the deal": [""], + "here are the key": ["the main"], + "here are a few": ["a few"], + "i hope this helps": [""], + "hope this helps": [""], + "feel free to": ["you can"], + "rest assured": [""], + "the good news is": [""], + "in this article, we'll": ["this covers"], + "in this guide, we'll": ["this covers"], + "buckle up": [""], + "you're not alone": [""], + "that being said,": ["still,", ""], + "when it comes to": ["for", "with"], +} + +# ── Русский ───────────────────────────────────────────────── +AI_CLICHES_RU: dict[str, list[str]] = { + "в современном мире стремительно развивающихся технологий": [ + "сегодня", "сейчас", ""], + "в современном быстро меняющемся мире": ["сегодня", "сейчас", ""], + "в современном цифровом мире": ["сегодня", "сейчас", ""], + "в современном мире": ["сегодня", "сейчас", ""], + "в современном обществе": ["сегодня", "сейчас", ""], + "в наши дни": ["сегодня", "сейчас", ""], + "в эпоху цифровых технологий": ["сейчас", "сегодня", ""], + "в условиях стремительного развития": ["на фоне быстрого роста", "пока всё быстро меняется"], + "стремительно развивающийся": ["быстрорастущий", "быстро меняющийся"], + "стремительно развивающихся": ["быстрорастущих", "быстро меняющихся"], + # Хеджирование + "важно отметить, что": ["отмечу, что", "заметим:", ""], + "следует отметить, что": ["отмечу, что", "заметим:", ""], + "необходимо отметить, что": ["отмечу, что", ""], + "стоит отметить, что": ["отмечу, что", "заметим:", ""], + "стоит подчеркнуть, что": ["подчеркну:", ""], + "нельзя не отметить": ["отмечу", "замечу"], + "важно понимать, что": ["поймите:", "суть в том, что", ""], + "необходимо учитывать, что": ["учтите:", ""], + # Штампы + # «роль в чём» — предложный падеж, замена его сохраняет + "играет ключевую роль в": ["многое решает в", "многое значит в"], + "играет важную роль в": ["многое значит в", "заметно сказывается в"], + "играет решающую роль в": ["решает дело в", "многое решает в"], + # «частью чего» — родительный падеж + "стал неотъемлемой частью": ["стал частью", "давно стал частью"], + "стала неотъемлемой частью": ["стала частью", "давно стала частью"], + "является неотъемлемой частью": ["остаётся частью", "давно стал частью"], + # «спектр чего» — родительный, поэтому «разные» не подходит + "широкий спектр": ["много", "множество", "масса"], + "широкий круг": ["много", "множество"], + "целый ряд": ["несколько", "много"], + "открывает новые горизонты": ["даёт новые возможности", "открывает новое"], + "открывает беспрецедентные возможности": ["даёт новые возможности", "открывает новое"], + "раскрывать беспрецедентные возможности": ["находить новые возможности"], + "позволяет раскрыть потенциал": ["помогает раскрыть потенциал"], + "в конечном итоге": ["в итоге", "в конце концов", ""], + "таким образом,": ["значит,", "выходит,", ""], + "подводя итог,": ["короче,", "итого:", ""], + "в заключение,": ["напоследок,", ""], + "в заключение": ["напоследок", ""], + # Ассистентский регистр + "отличный вопрос": ["", "хороший вопрос"], + "отличный вопрос!": [""], + "давайте разберёмся": ["разберёмся"], + "давайте разберемся": ["разберёмся"], + "давайте рассмотрим": ["рассмотрим"], + "надеюсь, это поможет": [""], + "надеюсь, это было полезно": [""], + "не переживайте": [""], + "вот в чём дело": [""], + "хорошая новость в том, что": [""], + "стоит помнить, что": ["помните:", ""], + "когда речь идёт о": ["для", "что касается"], + "когда дело доходит до": ["для", "что касается"], + "поиск правильного баланса": ["баланс", "поиск баланса"], + # «изучить что» — винительный; «разобраться в чём» его бы сломало + "более глубоко изучить": ["изучить глубже", "внимательнее изучить"], + "необходимо более глубоко изучить": ["стоит изучить глубже", "надо изучить внимательнее"], +} + +# ── Украинский ────────────────────────────────────────────── +AI_CLICHES_UK: dict[str, list[str]] = { + "у сучасному світі стрімкого розвитку технологій": ["сьогодні", "зараз", ""], + "у сучасному швидкозмінному світі": ["сьогодні", "зараз", ""], + "у сучасному цифровому світі": ["сьогодні", "зараз", ""], + "у сучасному світі": ["сьогодні", "зараз", ""], + "у сучасному суспільстві": ["сьогодні", "зараз", ""], + "в епоху цифрових технологій": ["зараз", "сьогодні", ""], + "стрімкого розвитку": ["швидкого зростання", "швидких змін"], + "стрімко розвивається": ["швидко зростає", "швидко змінюється"], + # Хеджування + "важливо зазначити, що": ["зазначу, що", "звернімо увагу:", ""], + "слід зазначити, що": ["зазначу, що", "звернімо увагу:", ""], + "варто зазначити, що": ["зазначу, що", ""], + "необхідно зазначити, що": ["зазначу, що", ""], + "варто підкреслити, що": ["підкреслю:", ""], + "важливо розуміти, що": ["зрозумійте:", "суть у тому, що", ""], + # Штампи + "відіграє ключову роль у": ["багато вирішує у", "багато важить у"], + "відіграє важливу роль у": ["багато важить у", "помітно позначається у"], + "став невід'ємною частиною": ["став частиною", "давно став частиною"], + "стала невід'ємною частиною": ["стала частиною", "давно стала частиною"], + "є невід'ємною частиною": ["залишається частиною", "давно став частиною"], + "широкий спектр": ["багато", "безліч"], + "низку переваг": ["кілька переваг", "чимало переваг"], + "відкриває нові горизонти": ["дає нові можливості", "відкриває нове"], + "зрештою,": ["врешті-решт,", ""], + "таким чином,": ["отже,", "виходить,", ""], + "підсумовуючи,": ["коротко:", ""], + "на завершення,": ["наостанок,", ""], + "на завершення": ["наостанок", ""], +} + +# ── Немецкий ──────────────────────────────────────────────── +AI_CLICHES_DE: dict[str, list[str]] = { + "in der heutigen schnelllebigen digitalen landschaft": ["heute", "derzeit", ""], + "in der heutigen sich schnell entwickelnden digitalen landschaft": [ + "heute", "derzeit", ""], + "in der heutigen digitalen welt": ["heute", "derzeit", ""], + "in der heutigen zeit": ["heute", "derzeit", ""], + "in der heutigen welt": ["heute", "derzeit", ""], + "in der heutigen gesellschaft": ["heute", "derzeit", ""], + "im digitalen zeitalter": ["heute", "derzeit", ""], + "sich schnell entwickelnden": ["schnell wachsenden", "sich wandelnden"], + # Absicherung + "es ist wichtig zu beachten, dass": ["beachte:", "wichtig:", ""], + "es ist wichtig zu betonen, dass": ["betont sei:", ""], + "es sei darauf hingewiesen, dass": ["übrigens:", ""], + "es ist erwähnenswert, dass": ["erwähnenswert:", ""], + # Floskeln + "spielt eine entscheidende rolle": ["ist entscheidend", "entscheidet viel"], + "spielt eine wichtige rolle": ["ist wichtig", "zählt"], + "spielt eine zentrale rolle": ["steht im Zentrum", "ist zentral"], + # «Bestandteil» верховодит родительным — замена его сохраняет + "ist ein integraler bestandteil": ["ist fester Bestandteil", "ist ein Teil"], + "eine vielzahl von": ["viele", "etliche"], + "ein breites spektrum an": ["viele", "etliche"], + "vielfältige auswirkungen": ["viele Folgen", "unterschiedliche Folgen"], + "vielfältigen auswirkungen": ["vielen Folgen", "unterschiedlichen Folgen"], + "eröffnet neue möglichkeiten": ["schafft neue Optionen", "öffnet Türen"], + "zusammenfassend lässt sich sagen, dass": ["kurz gesagt:", "unterm Strich:", ""], + "zusammenfassend": ["kurz gesagt", "unterm Strich", ""], + "letztendlich": ["am Ende", ""], + "abschließend": ["zum Schluss", ""], +} + +# ── Испанский ─────────────────────────────────────────────── +AI_CLICHES_ES: dict[str, list[str]] = { + "en el mundo actual de tecnologías en rápida evolución": ["hoy", "ahora", ""], + "en el mundo actual, en rápida evolución": ["hoy", "ahora", ""], + "en el mundo digital actual": ["hoy", "ahora", ""], + "en el mundo actual": ["hoy", "ahora", ""], + "en la sociedad actual": ["hoy", "ahora", ""], + "en la era digital": ["hoy", "ahora", ""], + "en rápida evolución": ["que cambia rápido", "en pleno cambio"], + # Matización + "es importante señalar que": ["ojo:", "conviene señalar que", ""], + "es importante destacar que": ["destaco que", "conviene destacar que", ""], + "cabe señalar que": ["señalo que", ""], + "cabe destacar que": ["destaco que", ""], + "es fundamental comprender que": ["hay que entender que", ""], + # Muletillas + "desempeña un papel crucial en": ["es clave en", "pesa mucho en"], + "desempeña un papel fundamental en": ["es clave en", "pesa mucho en"], + "juega un papel importante en": ["es importante en", "cuenta en"], + "se ha convertido en una parte integral de": ["ya forma parte de", "ya está dentro de"], + "es una parte integral de": ["forma parte de", "está dentro de"], + "una amplia gama de": ["muchos", "todo tipo de"], + "una amplia variedad de": ["muchos", "todo tipo de"], + "un sinfín de": ["muchísimos", "un montón de"], + "abre nuevas oportunidades": ["abre puertas", "crea opciones nuevas"], + "en conclusión,": ["en resumen,", "total,", ""], + "en resumen,": ["resumiendo,", ""], + "en última instancia,": ["al final,", ""], + "finalmente,": ["por último,", ""], +} + +# ── Польский ──────────────────────────────────────────────── +AI_CLICHES_PL: dict[str, list[str]] = { + "w dzisiejszym szybko zmieniającym się świecie cyfrowym": ["dziś", "teraz", ""], + "w dzisiejszym szybko zmieniającym się świecie": ["dziś", "teraz", ""], + "w dzisiejszym cyfrowym świecie": ["dziś", "teraz", ""], + "w dzisiejszym świecie": ["dziś", "teraz", ""], + "w dzisiejszych czasach": ["dziś", "teraz", ""], + "w erze cyfrowej": ["dziś", "teraz", ""], + "szybko rozwijający się": ["szybko rosnący", "zmieniający się"], + # Asekuracja + "warto zauważyć, że": ["zauważmy:", "warto dodać:", ""], + "należy zauważyć, że": ["zauważmy:", ""], + "warto podkreślić, że": ["podkreślę:", ""], + "należy podkreślić, że": ["podkreślę:", ""], + "ważne jest, aby zrozumieć, że": ["trzeba zrozumieć, że", ""], + # Frazesy + "odgrywa kluczową rolę w": ["wiele znaczy w", "wiele decyduje w"], + "odgrywa istotną rolę w": ["liczy się w", "wiele znaczy w"], + "stał się nieodłączną częścią": ["stał się częścią", "od dawna jest częścią"], + "stała się nieodłączną częścią": ["stała się częścią", "od dawna jest częścią"], + "jest nieodłączną częścią": ["należy do", "jest częścią"], + "szeroki zakres": ["wiele", "mnóstwo"], + "szeroką gamę": ["wiele", "mnóstwo"], + "otwiera nowe możliwości": ["daje nowe opcje", "otwiera drzwi"], + "podsumowując,": ["krótko mówiąc,", "w skrócie,", ""], + "ostatecznie,": ["w końcu,", ""], + "na zakończenie,": ["na koniec,", ""], +} + +AI_CLICHES: dict[str, dict[str, list[str]]] = { + "en": AI_CLICHES_EN, + "ru": AI_CLICHES_RU, + "uk": AI_CLICHES_UK, + "de": AI_CLICHES_DE, + "es": AI_CLICHES_ES, + "pl": AI_CLICHES_PL, +} + + +def merge_into(pack: dict) -> dict: + """Подмешать клише языка в ``bureaucratic_phrases`` пакета. + + Существующие записи пакета имеют приоритет: они выверены дольше и + могут быть точнее для конкретного языка. + """ + cliches = AI_CLICHES.get(pack.get("code", "")) + if not cliches: + return pack + phrases = dict(pack.get("bureaucratic_phrases") or {}) + for key, alts in cliches.items(): + phrases.setdefault(key, list(alts)) + pack["bureaucratic_phrases"] = phrases + return pack diff --git a/texthumanize/media_watermark.py b/texthumanize/media_watermark.py index 1d0baa3..05f4b49 100644 --- a/texthumanize/media_watermark.py +++ b/texthumanize/media_watermark.py @@ -69,6 +69,15 @@ "elevenlabs": "ElevenLabs (audio)", "suno": "Suno (audio)", "udio": "Udio (audio)", + # 2026 additions — distinctive tokens only. + "recraft": "Recraft", + "seedream": "Seedream (ByteDance)", + "seedance": "Seedance (ByteDance)", + "hunyuanvideo": "Hunyuan (Tencent)", + "nano-banana": "Gemini Nano Banana (Google)", + "reve.art": "Reve", + "stable-signature": "Meta Stable Signature (declared)", + "trainedalgorithmic": "AI-generated (IPTC digitalSourceType)", } # C2PA / provenance standard markers (case-insensitive byte search). @@ -197,10 +206,25 @@ def _iter_png_chunks(data: bytes): _PNG_TEXT_CHUNKS = {b"tEXt", b"zTXt", b"iTXt"} -_PNG_AI_TEXT_KEYS = { - "parameters", "prompt", "workflow", "negative prompt", "comment", - "software", "sd-metadata", "dream", "title", "description", +# Chunk keywords essentially unique to image-generation tooling — the key name +# alone is conclusive. Maps key -> the tool it identifies. +_PNG_STRONG_KEYS = { + "parameters": "AUTOMATIC1111 / Forge / Fooocus (Stable Diffusion)", + "workflow": "ComfyUI (Stable Diffusion)", + "sd-metadata": "InvokeAI (Stable Diffusion)", + "invokeai_metadata": "InvokeAI (Stable Diffusion)", + "sui_image_params": "SwarmUI (Stable Diffusion)", + "dream": "InvokeAI (Stable Diffusion)", + "negative prompt": "Stable Diffusion", } +# Generic keywords (prompt/comment/software/title/description/author) are NOT +# treated as AI on the key name alone — a camera caption in Description or a +# Photoshop Comment used to be misread as AI. They count only when the VALUE +# carries an actual generation-parameter hint below. +_PNG_PARAM_HINTS = ( + "steps:", "sampler", "cfg scale", "model hash", "denoising_strength", + "class_type", "sampler_name", '"scheduler"', "negative prompt:", +) def _parse_png(data: bytes) -> tuple[list[dict[str, Any]], dict[str, Any]]: @@ -211,12 +235,17 @@ def _parse_png(data: bytes) -> tuple[list[dict[str, Any]], dict[str, Any]]: text = payload.replace(b"\x00", b" ").decode("latin-1", "replace") key = text.split(" ", 1)[0].strip().lower() meta["text_chunks"].append(text[:200]) - if key in _PNG_AI_TEXT_KEYS or any(k in text.lower() for k in ("steps:", "sampler", "cfg scale", "seed:", "model hash")): + strong = _PNG_STRONG_KEYS.get(key) + has_hint = any(k in text.lower() for k in _PNG_PARAM_HINTS) + if strong or has_hint: findings.append({ "type": "embedded_generation_parameters", "category": "generation_params", "severity": "high", - "detail": f"PNG {ctype.decode()} chunk with generation metadata ('{key}')", + "detail": (f"PNG {ctype.decode()} '{key}' chunk — {strong}" + if strong else + f"PNG {ctype.decode()} chunk with Stable Diffusion generation parameters"), + **({"generator": strong} if strong else {}), }) findings.extend(_scan_markers(payload)) elif ctype in (b"eXIf", b"caBX", b"iDOT"): diff --git a/training/README.md b/training/README.md new file mode 100644 index 0000000..e523d1d --- /dev/null +++ b/training/README.md @@ -0,0 +1,73 @@ +# Self-improving detector weights + +TextHumanize's AI-text detector is a **transparent learned model**, not a hand- +wavy "neural net" and not a black box. Every text becomes a vector of readable +metric scores — `pattern`, `burstiness`, `structure`, `voice`, `stylometry`, … — +and the model is simply the per-metric weight applied to that vector. That is a +single-layer logistic classifier (one neuron); its parameters are the weights in +[`texthumanize/detector_weights.json`](../texthumanize/detector_weights.json). + +We fit those weights **offline** from a labelled corpus and ship only the fitted +numbers. The library carries no ML runtime, downloads no model, and every +parameter is readable and diffable in git. This is what lets the detector get +**better with each release** while staying tiny and fully offline. + +## The loop + +``` +training/corpus.jsonl → train_weights.py → detector_weights.json → detector + (labelled data) (numpy fit + gate) (versioned weights) (uses them) + ▲ │ + └────────────── new labels grow the corpus, next refit improves ◄────────┘ +``` + +1. **Corpus** — `corpus.jsonl`, one object per line: `{"text", "label":"ai"|"human", "lang"}`. + Seeded with public-domain human prose (guaranteed human) and characteristic + AI patterns across en/ru/uk/de/es/pl. **It grows over time** — that is the + "learning". +2. **Fit** — `train_weights.py` runs the detector to extract each sample's metric + vector, then fits non-negative weights on the probability simplex with a + projected-gradient logistic objective. +3. **Guardrails** so an automated refit can never quietly regress: + - **Non-negative + simplex** — preserves the "each metric votes toward AI" + meaning of the ensemble. + - **Shrinkage by evidence** — the fit is trusted only in proportion to corpus + size (`alpha = min(1, n/trust_n)`, `trust_n=500`). With today's small corpus + the candidate barely moves off the shipped weights; as the corpus grows past + `trust_n` the data takes over. This is the mechanism by which more labels = + a genuinely better model, without a lucky small split zeroing out a strong + metric. + - **Held-out gate** — a candidate is only accepted if it does not regress + held-out accuracy/AUROC versus the current weights. +4. **Ship** — an accepted candidate is written to `detector_weights.json`; the + detector loads it (falling back to the hand-tuned defaults if absent/invalid). + +## Run it + +```bash +python training/train_weights.py # fit + report, writes nothing +python training/train_weights.py --write # write the candidate iff the gate passes +``` + +## Contributing labels + +The most useful contribution is **more, cleaner labelled data**. Add lines to +`corpus.jsonl` — real human writing (ideally with a verifiable public-domain or +self-authored source) and real AI output, tagged by language. Keep classes and +languages roughly balanced. Open a PR; CI re-fits and reports the effect. + +## Privacy — what the corpus is NOT + +This corpus is **curated and public / explicitly contributed**. It is **not** +harvested from users. The browser extension's 👍/👎 detection feedback is stored +on-device and is **content-free** (it records the outcome and structural hints, +never your text), so there is no user text to ingest here even in principle. Site +detection has a parallel, structural-only feedback path (see the extension's +`engine/site-forensics.js`) — those hints carry no personal content either. + +## Automated retraining + +`.github/workflows/retrain.yml` runs the fit on a schedule / manual dispatch / +when `corpus.jsonl` changes. If the gate passes and the weights actually move, it +opens a pull request with the new `detector_weights.json` and the training +report. A human reviews and merges — the model never updates itself unattended. diff --git a/training/corpus.jsonl b/training/corpus.jsonl new file mode 100644 index 0000000..3d865ed --- /dev/null +++ b/training/corpus.jsonl @@ -0,0 +1,33 @@ +{"label": "human", "lang": "en", "source": "public-domain:austen", "text": "It is a truth universally acknowledged, that a single man in possession of a good fortune, must be in want of a wife. However little known the feelings or views of such a man may be on his first entering a neighbourhood, this truth is so well fixed in the minds of the surrounding families, that he is considered as the rightful property of some one or other of their daughters. My dear Mr. Bennet, said his lady to him one day, have you heard that Netherfield Park is let at last? Mr. Bennet replied that he had not."} +{"label": "human", "lang": "en", "source": "public-domain:twain", "text": "You don't know about me without you have read a book by the name of The Adventures of Tom Sawyer; but that ain't no matter. That book was made by Mr. Mark Twain, and he told the truth, mainly. There was things which he stretched, but mainly he told the truth. That is nothing. I never seen anybody but lied one time or another, without it was Aunt Polly, or the widow, or maybe Mary."} +{"label": "human", "lang": "en", "source": "public-domain:dickens", "text": "It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness. We had everything before us, we had nothing before us. In short, the period was so far like the present period."} +{"label": "human", "lang": "en", "source": "authored:forum", "text": "ok so quick update on the deck build. got the joists in saturday, and of course it rained sunday so everything sat under a tarp. the inspector comes tuesday. one thing nobody tells you: the ledger flashing is the part that eats your whole weekend, not the framing. i went through two boxes of screws because the first ones were garbage and snapped at the head. anyway. if your posts are even slightly out of plumb fix it NOW, not after the beams go on."} +{"label": "human", "lang": "en", "source": "authored:postmortem", "text": "The migration took three weekends instead of one. Our old cluster ran Postgres 11 with a pile of hand-written triggers nobody had touched since 2019, and two of them silently depended on a locale setting the new boxes did not share. We found that out the bad way: order totals off by a cent in some locales, only on refunds. The fix itself was four lines. Finding it was two days of diffing WAL dumps."} +{"label": "human", "lang": "en", "source": "authored:review", "text": "Bought these boots in November and wore them through a genuinely miserable winter. The left one started letting water in around the seam by February, which is annoying for the price. Customer service replaced them, no argument, so I can't complain there. Sizing runs big, order half down. The laces are garbage though, first thing I did was swap them."} +{"label": "ai", "lang": "en", "source": "pattern:corporate", "text": "In today's rapidly evolving digital landscape, it is important to note that leveraging synergistic solutions can significantly enhance productivity. Furthermore, organizations must carefully consider the multifaceted implications of these transformative technologies. Moreover, it should be noted that the implementation of such systems requires careful consideration of numerous factors. In conclusion, the utilization of robust frameworks plays a crucial role in navigating the complexities of modern business environments."} +{"label": "ai", "lang": "en", "source": "pattern:assistant", "text": "Great question! Let's break down how solar panels work. Essentially, solar panels convert sunlight into electricity through the photovoltaic effect. When sunlight hits the panel, it excites electrons in the silicon cells, creating an electric current. Here are the key components: First, the panels themselves capture sunlight. Second, an inverter converts the direct current into alternating current. Third, your home's electrical panel distributes the power. It's worth noting that modern panels are remarkably efficient. Ultimately, solar power offers a sustainable and cost-effective solution."} +{"label": "ai", "lang": "en", "source": "pattern:listicle", "text": "Artificial intelligence has become an integral part of our daily lives. It is worth noting that these powerful tools offer a wide range of benefits across various industries. Additionally, the seamless integration of machine learning algorithms enables businesses to unlock unprecedented opportunities. However, it is essential to delve deeper into the ethical considerations. Ultimately, striking the right balance remains pivotal for sustainable growth."} +{"label": "ai", "lang": "en", "source": "pattern:academic", "text": "The proliferation of digital technologies has fundamentally transformed contemporary society. This transformation is characterized by the widespread adoption of interconnected systems. The implementation of these frameworks necessitates comprehensive evaluation. Furthermore, the utilization of advanced methodologies facilitates the optimization of organizational processes. It is important to note that such developments require careful consideration of both technical and social factors."} +{"label": "ai", "lang": "en", "source": "pattern:seo", "text": "When it comes to choosing the right running shoes, there are several important factors to consider. First and foremost, it is essential to understand your foot type. Additionally, the terrain you run on plays a crucial role in determining the ideal shoe. Moreover, comfort should never be overlooked. In conclusion, by carefully considering these factors, you can unlock the full potential of your running experience and achieve your fitness goals."} +{"label": "ai", "lang": "en", "source": "pattern:marketing", "text": "Unlock the power of seamless collaboration with our cutting-edge platform. Designed to streamline your workflow, our innovative solution empowers teams to achieve unprecedented levels of productivity. Whether you are a small startup or a large enterprise, our comprehensive suite of tools caters to a wide range of needs. Experience the future of work today and transform the way your organization operates."} +{"label": "human", "lang": "ru", "source": "public-domain:chekhov", "text": "Говорили, что на набережной появилось новое лицо: дама с собачкой. Дмитрий Дмитрич Гуров, проживший в Ялте уже две недели и привыкший тут, тоже стал интересоваться новыми лицами. Сидя в павильоне у Верне, он видел, как по набережной прошла молодая дама, невысокого роста блондинка, в берете; за нею бежал белый шпиц. И потом он встречал её в городском саду и на сквере по нескольку раз в день."} +{"label": "human", "lang": "ru", "source": "public-domain:tolstoy", "text": "Все счастливые семьи похожи друг на друга, каждая несчастливая семья несчастлива по-своему. Всё смешалось в доме Облонских. Жена узнала, что муж был в связи с бывшею в их доме француженкою-гувернанткой, и объявила мужу, что не может жить с ним в одном доме. Положение это продолжалось уже третий день и мучительно чувствовалось и самими супругами, и всеми членами семьи."} +{"label": "human", "lang": "ru", "source": "authored:forum", "text": "короче, поменял я этот датчик. три часа, два содранных пальца и одна потерянная головка на десять. кто проектировал этот моторный отсек — отдельный привет ему. по деньгам: датчик 1800, прокладка 200, и ещё герметик у меня был. на сервисе просили пять с половиной, так что вроде сэкономил, но по времени конечно ад. да, важное: не берите аналог за 900, у меня такой сдох через месяц."} +{"label": "human", "lang": "ru", "source": "authored:report", "text": "Ремонт моста на Садовой опять перенесли, теперь на сентябрь. Подрядчик объясняет задержку тем, что при вскрытии опор нашли трещины, которых не было в проектной документации две тысячи восьмого года. Жители соседних домов жалуются в первую очередь не на сроки, а на объезд: автобусы идут через узкую Полевую, и по утрам там стоит всё намертво."} +{"label": "ai", "lang": "ru", "source": "pattern:corporate", "text": "В современном мире стремительно развивающихся технологий важно отметить, что использование синергетических решений может значительно повысить производительность. Кроме того, организациям следует тщательно учитывать многогранные последствия этих трансформационных технологий. Более того, следует отметить, что внедрение подобных систем требует внимательного рассмотрения множества факторов. Таким образом, использование надёжных решений играет ключевую роль в достижении устойчивого успеха."} +{"label": "ai", "lang": "ru", "source": "pattern:assistant", "text": "Отличный вопрос! Давайте разберёмся, как работают солнечные панели. По сути, солнечные панели преобразуют солнечный свет в электричество благодаря фотоэлектрическому эффекту. Когда свет попадает на панель, он возбуждает электроны в кремниевых ячейках. Вот ключевые компоненты: во-первых, сами панели улавливают свет. Во-вторых, инвертор преобразует ток. В-третьих, электрощит распределяет энергию. Стоит отметить, что современные панели удивительно эффективны. В конечном итоге, солнечная энергия предлагает устойчивое решение."} +{"label": "ai", "lang": "ru", "source": "pattern:article", "text": "Искусственный интеллект стал неотъемлемой частью нашей повседневной жизни. Стоит отметить, что эти мощные инструменты предоставляют широкий спектр преимуществ в различных отраслях. Кроме того, бесшовная интеграция алгоритмов машинного обучения позволяет компаниям раскрывать беспрецедентные возможности. Однако необходимо более глубоко изучить этические аспекты. В конечном итоге, поиск правильного баланса остаётся ключевым фактором устойчивого развития."} +{"label": "ai", "lang": "ru", "source": "pattern:seo", "text": "Когда речь идёт о выборе правильных беговых кроссовок, существует несколько важных факторов, которые следует учитывать. Прежде всего, важно понимать тип вашей стопы. Кроме того, поверхность, по которой вы бегаете, играет ключевую роль. Более того, комфорт никогда не следует упускать из виду. Таким образом, тщательно учитывая эти факторы, вы сможете раскрыть весь потенциал."} +{"label": "human", "lang": "uk", "source": "public-domain:kotsiubynsky", "text": "Іван був дев'ятнадцятою дитиною в гуцульській родині Палійчуків. Двадцятою і останньою була Анничка. Не знати, чи то вічний шум Черемошу і скарги гірських потоків, що сповняли самотню хату на високій кичері, чи сум чорних смерекових лісів лякав дитину, тільки Іван все плакав, кричав по ночах, погано ріс і дивився на неню таким глибоким, старече розумним зором."} +{"label": "human", "lang": "uk", "source": "authored:forum", "text": "коротко: поміняв я той датчик нарешті. три години і два обдертих пальці. хто конструював цей моторний відсік — окремий привіт. по грошах вийшло десь дві тисячі за все, на сервісі правили більше ніж удвічі. головне не беріть дешевий аналог, у сусіда такий здох за місяць, а оригінал ходить рік вже."} +{"label": "ai", "lang": "uk", "source": "pattern:corporate", "text": "У сучасному світі стрімкого розвитку технологій важливо зазначити, що використання синергетичних рішень може значно підвищити продуктивність. Крім того, організаціям слід ретельно враховувати багатогранні наслідки цих трансформаційних технологій. Більш того, слід зазначити, що впровадження подібних систем потребує уважного розгляду численних факторів. Таким чином, використання надійних рішень відіграє ключову роль."} +{"label": "ai", "lang": "uk", "source": "pattern:article", "text": "Штучний інтелект став невід'ємною частиною нашого повсякденного життя. Варто зазначити, що ці потужні інструменти надають широкий спектр переваг у різних галузях. Крім того, безшовна інтеграція алгоритмів машинного навчання дозволяє компаніям розкривати безпрецедентні можливості. Однак необхідно глибше вивчити етичні аспекти. Зрештою, пошук правильного балансу залишається ключовим фактором."} +{"label": "human", "lang": "de", "source": "public-domain:kafka", "text": "Als Gregor Samsa eines Morgens aus unruhigen Träumen erwachte, fand er sich in seinem Bett zu einem ungeheueren Ungeziefer verwandelt. Er lag auf seinem panzerartig harten Rücken und sah, wenn er den Kopf ein wenig hob, seinen gewölbten, braunen, von bogenförmigen Versteifungen geteilten Bauch, auf dessen Höhe sich die Bettdecke, zum gänzlichen Niedergleiten bereit, kaum noch erhalten konnte."} +{"label": "human", "lang": "de", "source": "authored:forum", "text": "kurzes update zum dachausbau. die sparren sind samstag reingekommen, sonntag hat es natürlich geregnet und alles lag unter einer plane. was dir keiner sagt: die verkleidung am rand frisst dein ganzes wochenende, nicht der rahmen. zwei schachteln schrauben durch, weil die ersten mist waren. egal. wenn die pfosten schief sind, jetzt richten, nicht nach den balken."} +{"label": "ai", "lang": "de", "source": "pattern:corporate", "text": "In der heutigen sich schnell entwickelnden digitalen Landschaft ist es wichtig zu beachten, dass die Nutzung synergetischer Lösungen die Produktivität erheblich steigern kann. Darüber hinaus müssen Organisationen die vielfältigen Auswirkungen dieser transformativen Technologien sorgfältig berücksichtigen. Zusammenfassend spielt der Einsatz robuster Frameworks eine entscheidende Rolle für den nachhaltigen Erfolg."} +{"label": "ai", "lang": "de", "source": "pattern:article", "text": "Künstliche Intelligenz ist zu einem festen Bestandteil unseres täglichen Lebens geworden. Es ist erwähnenswert, dass diese leistungsstarken Werkzeuge eine Vielzahl von Vorteilen in verschiedenen Branchen bieten. Darüber hinaus ermöglicht die nahtlose Integration von Algorithmen den Unternehmen, beispiellose Möglichkeiten zu erschließen. Letztendlich bleibt das richtige Gleichgewicht entscheidend."} +{"label": "human", "lang": "es", "source": "public-domain:cervantes", "text": "En un lugar de la Mancha, de cuyo nombre no quiero acordarme, no ha mucho tiempo que vivía un hidalgo de los de lanza en astillero, adarga antigua, rocín flaco y galgo corredor. Una olla de algo más vaca que carnero, salpicón las más noches, duelos y quebrantos los sábados, lentejas los viernes, algún palomino de añadidura los domingos, consumían las tres partes de su hacienda."} +{"label": "ai", "lang": "es", "source": "pattern:corporate", "text": "En el mundo actual de tecnologías en rápida evolución, es importante señalar que el aprovechamiento de soluciones sinérgicas puede mejorar significativamente la productividad. Además, las organizaciones deben considerar cuidadosamente las implicaciones multifacéticas de estas tecnologías transformadoras. En conclusión, la utilización de marcos robustos desempeña un papel crucial en el éxito sostenible."} +{"label": "ai", "lang": "es", "source": "pattern:article", "text": "La inteligencia artificial se ha convertido en una parte integral de nuestra vida diaria. Cabe señalar que estas potentes herramientas ofrecen una amplia gama de beneficios en diversas industrias. Además, la integración perfecta de los algoritmos permite a las empresas desbloquear oportunidades sin precedentes. En última instancia, lograr el equilibrio adecuado sigue siendo fundamental."} +{"label": "human", "lang": "pl", "source": "authored:forum", "text": "krótka aktualizacja budowy tarasu. legary weszły w sobotę, w niedzielę oczywiście padało i wszystko leżało pod plandeką. czego nikt ci nie mówi: obróbka przy krawędzi zjada cały weekend, nie konstrukcja. przeszedłem przez dwa pudełka wkrętów, bo pierwsze były do niczego. jak słupki są krzywe, prostuj teraz, nie po belkach."} +{"label": "ai", "lang": "pl", "source": "pattern:corporate", "text": "W dzisiejszym szybko zmieniającym się cyfrowym świecie warto zauważyć, że wykorzystanie synergicznych rozwiązań może znacząco zwiększyć produktywność. Ponadto organizacje muszą starannie rozważyć wieloaspektowe konsekwencje tych transformacyjnych technologii. Podsumowując, wykorzystanie solidnych struktur odgrywa kluczową rolę w zrównoważonym sukcesie."} diff --git a/training/train_weights.py b/training/train_weights.py new file mode 100644 index 0000000..aa097ce --- /dev/null +++ b/training/train_weights.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Fit the detector's metric weights from a labelled corpus. + +This is TextHumanize's "learned model" — and it is deliberately a *transparent* +one. The detector turns each text into a vector of interpretable metric scores +(pattern, burstiness, structure, voice, …); the model is just the per-metric +weight applied to that vector. That is a single-layer logistic classifier — a +one-neuron net — whose parameters are these weights. We fit it OFFLINE here and +ship only the fitted numbers (``texthumanize/detector_weights.json``): no model +runtime, no black box, every parameter readable and diffable in git. + +Guardrails baked in so an automated refit can never quietly make things worse: + * weights are constrained non-negative and to the probability simplex, so the + "each metric votes toward AI" meaning of the ensemble is preserved; + * the fit is L2-regularised toward the current shipped weights, so a small or + skewed corpus nudges rather than overwrites; + * a held-out split gates the result — a candidate is only recommended when it + does not regress held-out accuracy/AUROC versus the current weights. + +Usage: + python training/train_weights.py # fit + report, no write + python training/train_weights.py --write # write if the gate passes + python training/train_weights.py --corpus x.jsonl --seed 0 + +Corpus format: one JSON object per line — {"text", "label": "ai"|"human", "lang"}. +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from texthumanize.detectors import AIDetector # noqa: E402 + +# Metric order = the keys of the detector's weight table. Kept explicit so the +# feature vector and the emitted weights line up 1:1 with the detector. +METRICS = list(AIDetector._WEIGHTS.keys()) + +# Attribute on DetectionResult for each metric (a few names differ from the key). +_ATTR = { + "pattern": "pattern_score", "burstiness": "burstiness_score", + "voice": "voice_score", "stylometry": "stylometry_score", + "entity": "entity_score", "structure": "structure_score", + "discourse": "discourse_score", "rhythm": "rhythm_score", + "entropy": "entropy_score", "opening": "opening_score", + "grammar": "grammar_score", "vocabulary": "vocabulary_score", + "perplexity": "perplexity_score", "semantic_rep": "semantic_rep_score", + "topic_sentence": "topic_sent_score", "coherence": "coherence_score", + "readability": "readability_score", "punctuation": "punctuation_score", + "zipf": "zipf_score", +} + + +def load_corpus(path: Path) -> list[dict]: + rows = [] + for line in path.read_text("utf-8").splitlines(): + line = line.strip() + if line: + rows.append(json.loads(line)) + return rows + + +def extract_features(rows: list[dict]) -> tuple[np.ndarray, np.ndarray]: + """Run the detector on each sample → feature matrix X and labels y.""" + det = AIDetector() + X, y = [], [] + for r in rows: + res = det.detect(r["text"], r.get("lang", "auto")) + X.append([getattr(res, _ATTR[m], 0.5) for m in METRICS]) + y.append(1.0 if r["label"] == "ai" else 0.0) + return np.asarray(X, float), np.asarray(y, float) + + +def project_to_simplex(v: np.ndarray) -> np.ndarray: + """Euclidean projection onto {w >= 0, sum w = 1} (Duchi et al. 2008).""" + u = np.sort(v)[::-1] + css = np.cumsum(u) - 1.0 + ind = np.arange(1, len(v) + 1) + cond = u - css / ind > 0 + rho = ind[cond][-1] + theta = css[cond][-1] / rho + return np.maximum(v - theta, 0.0) + + +def fit(X, y, w0, l2=6.0, iters=4000, lr=0.4, seed=0): + """Projected-gradient logistic fit of simplex weights + scale/bias. + + Minimises BCE(sigmoid(a* + b), y) + l2 * ||w - w0||^2, with w kept on + the simplex. Only w is shipped; a,b emulate the detector's own calibration + during fitting so the weights land in a usable range. + """ + rng = np.random.default_rng(seed) + w = w0.copy() + a, b = 8.0, -4.0 + n = len(y) + for _ in range(iters): + m = X @ w + z = a * m + b + p = 1.0 / (1.0 + np.exp(-np.clip(z, -30, 30))) + err = p - y + gw = (X.T @ (err * a)) / n + 2.0 * l2 * (w - w0) + ga = float((err * m).mean()) + gb = float(err.mean()) + w = project_to_simplex(w - lr * gw) + a -= lr * ga + b -= lr * gb + _ = rng # reserved for optional stochastic variants + return w, a, b + + +def evaluate(rows, weights): + """Accuracy + AUROC on the FULL detector with `weights` installed.""" + AIDetector._fitted_weights_cache = dict(weights) + det = AIDetector() + probs, y = [], [] + for r in rows: + probs.append(det.detect(r["text"], r.get("lang", "auto")).ai_probability) + y.append(1 if r["label"] == "ai" else 0) + AIDetector._fitted_weights_cache = None # reset + probs, y = np.asarray(probs), np.asarray(y) + acc = float(((probs >= 0.5).astype(int) == y).mean()) + auroc = _auroc(y, probs) + return acc, auroc + + +def _auroc(y, s): + pos = s[y == 1] + neg = s[y == 0] + if len(pos) == 0 or len(neg) == 0: + return float("nan") + wins = (pos[:, None] > neg[None, :]).sum() + 0.5 * (pos[:, None] == neg[None, :]).sum() + return float(wins) / (len(pos) * len(neg)) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--corpus", default=str(Path(__file__).parent / "corpus.jsonl")) + ap.add_argument("--out", default=str(ROOT / "texthumanize" / "detector_weights.json")) + ap.add_argument("--l2", type=float, default=6.0) + ap.add_argument("--trust-n", type=int, default=500, + help="corpus size at which the fit is fully trusted (shrinkage)") + ap.add_argument("--seed", type=int, default=0) + ap.add_argument("--write", action="store_true", help="write weights if the held-out gate passes") + args = ap.parse_args() + + rows = load_corpus(Path(args.corpus)) + n_ai = sum(1 for r in rows if r["label"] == "ai") + print(f"corpus: {len(rows)} samples ({n_ai} ai / {len(rows) - n_ai} human)") + + X, y = extract_features(rows) + w0 = np.array([AIDetector._WEIGHTS[m] for m in METRICS], float) + w0 = w0 / w0.sum() + + # Deterministic held-out split (every 3rd sample), stratified enough for a gate. + idx = np.arange(len(rows)) + held = idx % 3 == 0 + train_mask = ~held + + w_fit, a, b = fit(X[train_mask], y[train_mask], w0, l2=args.l2, seed=args.seed) + + # Shrinkage by evidence: trust the fit only in proportion to how much + # labelled data backs it. With a tiny corpus the candidate barely moves off + # the shipped weights (so a lucky separable split can't zero out a strong + # metric like `pattern`); as the corpus grows past TRUST_N the fit takes + # over. This is the mechanism by which the model genuinely improves as the + # community contributes more labels. + n_train = int(train_mask.sum()) + alpha = min(1.0, n_train / args.trust_n) + w = project_to_simplex(alpha * w_fit + (1.0 - alpha) * w0) + print(f"shrinkage: alpha={alpha:.3f} (n_train={n_train}, trust_n={args.trust_n})") + + cur = {m: AIDetector._WEIGHTS[m] for m in METRICS} + cand = {m: round(float(wi), 5) for m, wi in zip(METRICS, w)} + + held_rows = [r for r, h in zip(rows, held) if h] + acc_cur, auroc_cur = evaluate(held_rows, cur) + acc_new, auroc_new = evaluate(held_rows, cand) + + print(f"\nheld-out ({len(held_rows)} samples):") + print(f" current acc={acc_cur:.3f} auroc={auroc_cur:.3f}") + print(f" candidate acc={acc_new:.3f} auroc={auroc_new:.3f}") + print("\ntop candidate weights:") + for m, wi in sorted(cand.items(), key=lambda kv: -kv[1])[:8]: + print(f" {m:14} {wi:.4f} (was {cur[m]:.4f})") + + # Gate: never regress held-out accuracy, and keep AUROC within a small margin. + passes = (acc_new >= acc_cur - 1e-9) and (auroc_new >= auroc_cur - 0.02) + print(f"\ngate: {'PASS' if passes else 'FAIL'} (candidate must not regress held-out)") + + if args.write: + if not passes: + print("not writing — gate failed.") + return 1 + payload = { + "schema": "texthumanize.detector_weights.v1", + "fitted": True, + "note": "Fitted by training/train_weights.py from training/corpus.jsonl. " + "Non-negative simplex weights, L2-regularised toward the previous " + "weights, gated on a held-out split. Retrain via the GitHub workflow.", + "trained_at": None, # stamped by the workflow (Date unavailable here) + "corpus_size": len(rows), + "metrics": {"held_out_accuracy": round(acc_new, 4), "held_out_auroc": round(auroc_new, 4)}, + "weights": cand, + } + Path(args.out).write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n") + print(f"wrote {args.out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())