diff --git a/CHANGELOG.md b/CHANGELOG.md index 72f5dc8..30665ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,14 @@ # Change Log All notable changes to this project will be documented in this file. -## [v4.0.0](https://github.com/fivefilters/readability.php/releases/tag/v4.0.0) +## [v4.0.0-beta.1](https://github.com/fivefilters/readability.php/releases/tag/v4.0.0-beta.1) Ground-up port from the latest Readability.js (v0.6.0) using Claude's Fable model. Uses PHP 8.4's new DOM API and native parser. See [UPGRADE.md](UPGRADE.md) for the full 3.x → 4.0 migration guide. ### Changed - Requires PHP >= 8.4; parsing and serialization use `Dom\HTMLDocument` (the WHATWG-spec Lexbor parser bundled with PHP), replacing HTML5-PHP and the legacy libxml path - `parse()` now returns a readonly `Article` value object (`title`, `content`, `textContent`, `length`, `excerpt`, `byline`, `siteName`, `dir`, `lang`, `publishedTime`, `image`, `images`, `contentElement`) and throws `ParseException` when no article is found -- `Configuration` is a readonly object with named constructor arguments; `maxTopCandidates` renamed to `nbTopCandidates` (matching Readability.js) +- Options are passed directly to the `Readability` constructor as named arguments, like the options object in Readability.js (`new Readability(fixRelativeURLs: true)`), and are all optional; a readonly `Configuration` object taking the same named arguments can be passed instead. `maxTopCandidates` renamed to `nbTopCandidates` (matching Readability.js) - Article output is wrapped in `
`, as in Readability.js - Byline is always extracted into `Article::$byline`; the `articleByline` option becomes `keepInlineByline`, which only controls whether an inline byline stays in the content (default removes it, as in Readability.js) - Image extraction moved onto the result object: `getImage()`/`getImages()` become `$article->image` / `$article->images` @@ -19,7 +19,7 @@ Ground-up port from the latest Readability.js (v0.6.0) using Claude's Fable mode ### Added - Parity with Readability.js 0.6.0: `lang` and `publishedTime` output; `maxElemsToParse`, `classesToPreserve`, `allowedVideoRegex`, `linkDensityModifier` and `debug` options; aria-modal dialog removal; ad/loading-indicator stripping; parsely/`article:author`/`itemprop` metadata sources; JSON-LD `@graph`, `@context`-object and array handling; Unicode comma scoring; updated regexes (mathjax, bilibili, en/em-dash title separators) - `Readerable::isProbablyReaderable()`, a port of Readability-readerable.js -- `parseDocument()` for callers who already hold a `Dom\HTMLDocument` +- `parse()` accepts an already-parsed `Dom\HTMLDocument` as well as an HTML string - Cross-check harness (`test/tools/`) that diffs this port's output against Readability.js over the whole corpus - Static analysis with [Psalm](https://psalm.dev/) (`composer analyse`), run in CI alongside the test suite diff --git a/README.md b/README.md index 6e31644..095e400 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,9 @@ PHP 8.4+, ext-dom, and ext-mbstring. First require the library using composer: -`composer require "fivefilters/readability.php:>=4.0"` +`composer require "fivefilters/readability.php:^4.0@beta"` + +Version 4.0 is currently in beta, so the `@beta` stability flag is needed; once the stable release is out, `composer require "fivefilters/readability.php:^4.0"` will do. Then create a Readability instance and feed `parse()` your HTML. It returns an `Article` object: @@ -28,10 +30,9 @@ Then create a Readability instance and feed `parse()` your HTML. It returns an ` contentElement->querySelectorAll('img[src]') as $img) { } ``` -If you already have a `\Dom\HTMLDocument` (for example because you want to pre-process it), use `parseDocument()` instead of `parse()`. Note that the document is modified in place while the article is extracted. +`parse()` also accepts a `\Dom\HTMLDocument` directly (for example because you want to pre-process it). Note that a passed document is modified in place while the article is extracted. ### Checking if a page is readerable @@ -91,7 +92,7 @@ use fivefilters\Readability\Readerable; // Only run the full parse if we suspect it will produce a meaningful result. if (Readerable::isProbablyReaderable($html)) { - $article = new Readability(new Configuration())->parse($html); + $article = new Readability()->parse($html); } ``` @@ -107,15 +108,24 @@ Readerable::isProbablyReaderable($html, minScore: 0, minContentLength: 120); ## Options -Configuration is a readonly object; pass options as named constructor arguments (or as an array via `Configuration::fromArray()`): +All options have defaults, so `new Readability()` is all you need for the standard behavior. To change something, pass options as named arguments — the equivalent of the options object in Readability.js: ```php -$configuration = new Configuration( +$readability = new Readability( fixRelativeURLs: true, originalURL: 'https://my.newspaper.url/article/something-interesting-to-read.html', ); ``` +If you want to build the options up separately, or share them between instances, you can also pass a `Configuration` — a readonly object taking the same named arguments (or an array via `Configuration::fromArray()`): + +```php +use fivefilters\Readability\Configuration; + +$configuration = new Configuration(fixRelativeURLs: true, originalURL: 'https://...'); +$readability = new Readability($configuration); +``` + Options matching Readability.js (same defaults): - **debug**: default `false`, log debug messages via `error_log()`. diff --git a/UPGRADE.md b/UPGRADE.md index ecb6687..d3ad6ae 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -11,9 +11,11 @@ Version 4.0 is a ground-up rewrite on PHP's native DOM extension (the Lexbor HTM | HTML parser | libxml or HTML5-PHP | native (Lexbor) | ```bash -composer require "fivefilters/readability.php:^4.0" +composer require "fivefilters/readability.php:^4.0@beta" ``` +(The `@beta` stability flag is needed while 4.0 is in beta; drop it once the stable release is out.) + ## The one-minute version **3.x** — `parse()` returns a bool and you read results off the Readability instance: @@ -33,7 +35,7 @@ try { **4.0** — `parse()` returns a readonly `Article` value object (or throws): ```php -$readability = new Readability(new Configuration()); +$readability = new Readability(); // options are named constructor arguments now, all optional try { $article = $readability->parse($html); @@ -79,7 +81,7 @@ $firstParagraph = $element->querySelector('p'); // CSS selectors work natively ## Configuration -3.x used an options array (or fluent setters). 4.0 uses a readonly object with named constructor arguments: +3.x required a `Configuration` built from an options array (or fluent setters). In 4.0, options are named arguments passed directly to `Readability` (like the options object in Readability.js), and they're all optional — `new Readability()` uses the defaults: ```php // 3.x @@ -88,15 +90,17 @@ $configuration = new Configuration([ 'originalURL' => 'https://example.com/article.html', ]); // or: $configuration->setFixRelativeURLs(true)->setOriginalURL('...'); +$readability = new Readability($configuration); // 4.0 -$configuration = new Configuration( +$readability = new Readability( fixRelativeURLs: true, originalURL: 'https://example.com/article.html', ); -// or: Configuration::fromArray(['fixRelativeURLs' => true, 'originalURL' => '...']) ``` +A readonly `Configuration` object still exists, taking the same named arguments, for options built up separately or shared between instances: `new Readability(new Configuration(fixRelativeURLs: true))`. To build one from an options array, use `Configuration::fromArray(['fixRelativeURLs' => true])`. + Option mapping: | 3.x | 4.0 | Notes | @@ -155,7 +159,7 @@ In 3.x, `articleByline` (default `false`) both enabled byline *detection* and, w ```php // Restore the 3.x default (byline stays in the content): -$configuration = new Configuration(keepInlineByline: true); +$readability = new Readability(keepInlineByline: true); ``` ### PSR-3 logging @@ -167,7 +171,7 @@ Still supported. Pass a PSR-3 `LoggerInterface` as the `logger` option instead o $configuration->setLogger($myLogger); // 4.0 -$configuration = new Configuration(logger: $myLogger); +$readability = new Readability(logger: $myLogger); ``` Debug messages are sent to the logger independently of the `debug` flag (which only controls `error_log()` output). @@ -175,7 +179,7 @@ Debug messages are sent to the logger independently of the `debug` flag (which o ## Removed features - `parser`, `substituteEntities`, `normalizeEntities`, `summonCthulhu` — all were libxml/HTML5-PHP workarounds and have no equivalent (the native Lexbor parser doesn't have the bugs they patched). -- `getDOMDocument(false)` — the whole-document variant is gone; `$article->contentElement` gives the extracted content only. If you need the full page, parse it yourself with `\Dom\HTMLDocument::createFromString()` and pass it to `parseDocument()`. +- `getDOMDocument(false)` — the whole-document variant is gone; `$article->contentElement` gives the extracted content only. If you need the full page, parse it yourself with `\Dom\HTMLDocument::createFromString()` and pass the document to `parse()`. - `getPathInfo()`, `loadHTML()`, `setExcerpt()` — internal helpers, not carried over. ## Behavior changes to be aware of @@ -197,6 +201,6 @@ Debug messages are sent to the logger independently of the `debug` flag (which o ## New in 4.0 - `Readerable::isProbablyReaderable($html)` — Mozilla's quick pre-check for whether a page is worth parsing, ported for the first time. -- `parseDocument(\Dom\HTMLDocument $document)` — parse a document you've already created (note: it's modified in place). +- `parse()` also accepts a `\Dom\HTMLDocument` you've already created, not just an HTML string (note: a passed document is modified in place). - `$article->textContent`, `->length`, `->lang`, `->publishedTime` outputs. - Metadata sources at Readability.js 0.6.0 parity: JSON-LD (`@graph`, `@context` objects), parsely, `article:author`, `itemprop`. diff --git a/docker/php/build.Dockerfile b/docker/php/build.Dockerfile deleted file mode 100644 index 21e9d4f..0000000 --- a/docker/php/build.Dockerfile +++ /dev/null @@ -1,62 +0,0 @@ -# Use this file to build a Docker image using the versions of PHP and Libxml specified. -# -# We have pre-built images at https://hub.docker.com/r/fivefilters/php-libxml which are -# faster to load than building from this file. -# -# To build using this file, type the following command from the root project folder -# (replace version of PHP/Libxml with the ones you want to use): -# -# docker build --build-arg PHP_VERSION=7.4 --build-arg LIBXML_VERSION=2.9.12 -t php-libxml -f ./docker/php/build.Dockerfile . - -# To upload the image to Docker Hub, the tag (-t) value should be something like org/repo:tag, e.g. for us, fivefilters/php-libxml:php-8-libxml-2.9.12 -# The tag can be applied afterwards too, e.g. docker tag php-libxml org/repo:tag - -ARG PHP_VERSION=8.4 -FROM php:${PHP_VERSION}-cli - -# Install sqlite and libonig-dev (required for building PHP 7.4), libreadline-dev for php 8.1 -RUN apt-get update && apt-get install -y libsqlite3-dev libonig-dev libreadline-dev -# Install libsodium (package doesn't work for some reason) -RUN curl https://download.libsodium.org/libsodium/releases/LATEST.tar.gz -o /tmp/libsodium.tar.gz && \ - cd /tmp && \ - tar -xzf libsodium.tar.gz && \ - cd libsodium-stable/ && \ - ./configure && \ - make && make check && \ - make install -# Install custom version of libxml2 -RUN apt-get install -y automake libtool unzip libssl-dev -# Remove current version -RUN apt-get remove -y libxml2 -# Download new version, configure and compile -ARG LIBXML_VERSION=2.9.14 -RUN curl https://gitlab.gnome.org/GNOME/libxml2/-/archive/v$LIBXML_VERSION/libxml2-v$LIBXML_VERSION.zip -o /tmp/libxml.zip && \ - cd /tmp && \ - unzip libxml.zip && \ - cd libxml2-v$LIBXML_VERSION && \ - ./autogen.sh --libdir=/usr/lib/x86_64-linux-gnu && \ - make && \ - make install -# Recompile PHP with the new libxml2 library -RUN docker-php-source extract && \ - cd /usr/src/php && \ - ./configure \ - --with-readline \ - --with-libxml \ - --enable-mbstring \ - --with-openssl \ - --with-config-file-path=/usr/local/etc/php \ - --with-config-file-scan-dir=/usr/local/etc/php/conf.d && \ - make && make install && \ - docker-php-source delete - -RUN apt-get update - -#RUN pecl install libsodium - -# Check if there's a pinned version of Xdebug for compatibility reasons -ARG XDEBUG_VERSION -RUN pecl install xdebug$(if [ ! ${XDEBUG_VERSION} = '' ]; then echo -${XDEBUG_VERSION} ; fi) && docker-php-ext-enable xdebug - -# Required by coveralls -RUN apt-get install git -y diff --git a/src/Readability.php b/src/Readability.php index 5c0c2c2..15e87b6 100644 --- a/src/Readability.php +++ b/src/Readability.php @@ -91,39 +91,49 @@ final class Readability private ?string $baseURI = null; private ?string $documentURI = null; - public function __construct(private readonly Configuration $configuration = new Configuration()) + private readonly Configuration $configuration; + + /** + * Options can be passed directly as named arguments — the PHP equivalent + * of Readability.js's options argument — e.g. + * new Readability(fixRelativeURLs: true, charThreshold: 20). + * See Configuration for the available options and their defaults. + * A pre-built Configuration is accepted too, for options built up + * separately or shared between instances. + * + * @param mixed ...$options Configuration options as named arguments + */ + public function __construct(?Configuration $configuration = null, mixed ...$options) { + if ($configuration !== null && $options !== []) { + throw new \InvalidArgumentException('Pass either a Configuration object or options as named arguments, not both.'); + } + $this->configuration = $configuration ?? Configuration::fromArray($options); $this->doc = \Dom\HTMLDocument::createEmpty(); $this->scores = new \SplObjectStorage(); $this->dataTables = new \SplObjectStorage(); } /** - * Parse an HTML string and return the article. + * Parse HTML — a string, or an already-parsed document — and return the + * article. A passed document is consumed: it is modified in place while + * the article is extracted. + * + * Mirrors Readability.js parse(). * * @throws ParseException when the input is empty, the document exceeds * maxElemsToParse, or no article content is found * (where Readability.js returns null) */ - public function parse(string $html): Article + public function parse(\Dom\HTMLDocument|string $document): Article { - if (trim($html) === '') { - throw ParseException::emptyInput(); + if (is_string($document)) { + if (trim($document) === '') { + throw ParseException::emptyInput(); + } + $document = \Dom\HTMLDocument::createFromString($document, LIBXML_NOERROR); } - return $this->parseDocument(\Dom\HTMLDocument::createFromString($html, LIBXML_NOERROR)); - } - - /** - * Runs readability on an already-parsed document. The document is - * consumed: it is modified in place while the article is extracted. - * - * Mirrors Readability.js parse(). - * - * @throws ParseException - */ - public function parseDocument(\Dom\HTMLDocument $document): Article - { $this->doc = $document; $this->scores = new \SplObjectStorage(); $this->dataTables = new \SplObjectStorage(); diff --git a/test/PhpFeaturesTest.php b/test/PhpFeaturesTest.php index 7d3ff6e..8287682 100644 --- a/test/PhpFeaturesTest.php +++ b/test/PhpFeaturesTest.php @@ -4,7 +4,6 @@ namespace fivefilters\Readability\Test; -use fivefilters\Readability\Configuration; use fivefilters\Readability\Readability; use PHPUnit\Framework\TestCase; use Psr\Log\AbstractLogger; @@ -26,7 +25,7 @@ private static function page(string $head = '', string $article = ''): string public function testLeadImageFromOgImage(): void { $html = self::page(''); - $article = new Readability(new Configuration())->parse($html); + $article = new Readability()->parse($html); $this->assertSame('https://example.com/lead.jpg', $article->image); $this->assertContains('https://example.com/lead.jpg', $article->images); } @@ -34,20 +33,20 @@ public function testLeadImageFromOgImage(): void public function testLeadImageFromTwitterImage(): void { $html = self::page(''); - $article = new Readability(new Configuration())->parse($html); + $article = new Readability()->parse($html); $this->assertSame('https://example.com/tw.jpg', $article->image); } public function testLeadImageFromLinkRel(): void { $html = self::page(''); - $article = new Readability(new Configuration())->parse($html); + $article = new Readability()->parse($html); $this->assertSame('https://example.com/link.jpg', $article->image); } public function testNoLeadImage(): void { - $article = new Readability(new Configuration())->parse(self::page()); + $article = new Readability()->parse(self::page()); $this->assertNull($article->image); $this->assertSame([], $article->images); } @@ -58,7 +57,7 @@ public function testImagesListCollectsContentImagesAndDeduplicates(): void '', '' ); - $article = new Readability(new Configuration())->parse($html); + $article = new Readability()->parse($html); // Lead image first, then content images, with the duplicate lead image collapsed. $this->assertSame([ 'https://example.com/lead.jpg', @@ -73,10 +72,10 @@ public function testImagesAreAbsolutizedWhenFixRelativeUrlsEnabled(): void '', '' ); - $article = new Readability(new Configuration( + $article = new Readability( fixRelativeURLs: true, originalURL: 'https://example.com/news/', - ))->parse($html); + )->parse($html); $this->assertSame('https://example.com/lead.jpg', $article->image); $this->assertSame([ 'https://example.com/lead.jpg', @@ -87,7 +86,7 @@ public function testImagesAreAbsolutizedWhenFixRelativeUrlsEnabled(): void public function testInlineBylineRemovedByDefault(): void { $html = self::page('', '

By Jane Doe

'); - $article = new Readability(new Configuration())->parse($html); + $article = new Readability()->parse($html); $this->assertSame('By Jane Doe', $article->byline); $this->assertStringNotContainsString('By Jane Doe', $article->content); } @@ -95,7 +94,7 @@ public function testInlineBylineRemovedByDefault(): void public function testInlineBylineKeptWhenConfigured(): void { $html = self::page('', '

By Jane Doe

'); - $article = new Readability(new Configuration(keepInlineByline: true))->parse($html); + $article = new Readability(keepInlineByline: true)->parse($html); // Still recorded as metadata... $this->assertSame('By Jane Doe', $article->byline); // ...but left in the content. @@ -114,7 +113,7 @@ public function log($level, string|\Stringable $message, array $context = []): v } }; - new Readability(new Configuration(logger: $logger))->parse(self::page()); + new Readability(logger: $logger)->parse(self::page()); $this->assertNotEmpty($logger->messages, 'logger should receive debug messages even with debug=false'); } @@ -123,7 +122,7 @@ public function testNoLoggingWithoutDebugOrLogger(): void { // With neither a logger nor the debug flag, log() is a no-op; this // parse should simply succeed without touching error_log. - $article = new Readability(new Configuration())->parse(self::page()); + $article = new Readability()->parse(self::page()); $this->assertNotSame('', $article->content); } } diff --git a/test/ReadabilityTest.php b/test/ReadabilityTest.php index 5fef650..3cbb7d6 100644 --- a/test/ReadabilityTest.php +++ b/test/ReadabilityTest.php @@ -92,31 +92,43 @@ public function testOversizedDocumentThrows(): void $this->expectException(ParseException::class); // html, head, body, div — parsing produces a full document $this->expectExceptionMessage('Aborting parsing document; 4 elements found'); - new Readability(new Configuration(maxElemsToParse: 1))->parse('
yo
'); + new Readability(maxElemsToParse: 1)->parse('
yo
'); } public function testCustomAllowedVideoRegex(): void { $source = '

Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc mollis leo lacus, vitae semper nisl ullamcorper ut.

' . ''; - $article = new Readability(new Configuration( + $article = new Readability( charThreshold: 20, allowedVideoRegex: '/.*mycustomdomain.com.*/', - ))->parse($source); + )->parse($source); $this->assertStringContainsString('