diff --git a/CLAUDE.md b/CLAUDE.md index 1e4499a7..8d528572 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,7 +82,7 @@ scheduler (every N minutes) or manual trigger via POST /api/jobs/:id/run |---|---|---| | Event bus | `lib/services/events/event-bus.js` | Plain `EventEmitter`; events: `jobs:runAll`, `jobs:runOne`, `jobs:status` | | SSE broker | `lib/services/sse/sse-broker.js` | Per-userId `Set`; heartbeat every 25s; pushes job status to UI | -| Similarity cache | `lib/services/similarity-check/` | In-memory SHA-256 Set; refreshes hourly; cross-provider dedup by title+price+address | +| Similarity cache | `lib/services/similarity-check/` | In-memory SHA-256 Set; refreshes hourly; per-job cross-provider dedup by title+price+address | | SqliteConnection | `lib/services/storage/SqliteConnection.js` | Singleton, WAL mode; `execute()`, `query()`, `withTransaction()` | | Migrations | `lib/services/storage/migrations/` | Numbered JS files each exporting `up(db)`; checksum-tracked in `schema_migrations` | | Extractor | `lib/services/extractor/` | Orchestrates Puppeteer + Cheerio; shared browser instance per job | @@ -117,4 +117,4 @@ Tools: `list_jobs`, `get_job`, `list_listings`, `get_listing`, `get_current_date - After building the task, run the tests - New features must be tested - New features must be properly documented with JsDoc -- You do **not** commit any changes, you do **not** create a new branch unless I told you so \ No newline at end of file +- You do **not** commit any changes, you do **not** create a new branch unless I told you so diff --git a/README.md b/README.md index 8b0790c9..a45ae88d 100755 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Finding an apartment or house in Germany can be stressful and time-consuming.\ **Fredy** makes it easier: it automatically scrapes **ImmoScout24, -Immowelt, Immonet, eBay Kleinanzeigen, and WG-Gesucht** and notifies you +Immowelt, Immonet, eBay Kleinanzeigen, WG-Gesucht, and InBerlinWohnen** and notifies you instantly via **Slack, Telegram, Email, ntfy, discord and more** when new listings appear. @@ -41,7 +41,7 @@ same listing twice. ## ✨ Key Features - 🏠 Scrapes **ImmoScout24, Immowelt, Immonet, eBay Kleinanzeigen, - WG-Gesucht** + WG-Gesucht, InBerlinWohnen** - ⚡ Instant notifications: Slack, Telegram, Email (SendGrid, Mailjet), ntfy, discord - 🔎 Uses the **ImmoScout Mobile API** (reverse engineered) diff --git a/lib/FredyPipelineExecutioner.js b/lib/FredyPipelineExecutioner.js index ace24888..3b9ce1d6 100755 --- a/lib/FredyPipelineExecutioner.js +++ b/lib/FredyPipelineExecutioner.js @@ -86,7 +86,11 @@ class FredyPipelineExecutioner { */ execute() { return Promise.resolve(urlModifier(this._providerConfig.url, this._providerConfig.sortByDateParam)) - .then(this._providerConfig.getListings?.bind(this) ?? this._getListings.bind(this)) + .then((url) => + this._providerConfig.getListings + ? this._providerConfig.getListings.call(this, url, this._browser) + : this._getListings(url), + ) .then(this._normalize.bind(this)) .then(this._filter.bind(this)) .then(this._findNew.bind(this)) @@ -439,6 +443,7 @@ class FredyPipelineExecutioner { const filteredIds = []; const keptListings = listings.filter((listing) => { const similar = this._similarityCache.checkAndAddEntry({ + jobId: this._jobKey, title: listing.title, address: listing.address, price: listing.price, diff --git a/lib/provider/inberlinwohnen.js b/lib/provider/inberlinwohnen.js new file mode 100644 index 00000000..1789904c --- /dev/null +++ b/lib/provider/inberlinwohnen.js @@ -0,0 +1,355 @@ +/* + * Copyright (c) 2026 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +import { buildHash, isOneOf } from '../utils.js'; +import { extractNumber } from '../utils/extract-number.js'; +import puppeteerExtractor from '../services/extractor/puppeteerExtractor.js'; +import logger from '../services/logger.js'; +import { load } from 'cheerio'; +/** @import { ParsedListing } from '../types/listing.js' */ +/** @import { ProviderConfig } from '../types/providerConfig.js' */ + +const BASE_URL = 'https://inberlinwohnen.de'; +const MAX_PAGES = 100; +const DETAIL_SELECTORS = { + 'berlinovo.de': [ + '.field--name-field-interior2', + '.field--name-field-interior', + '.field--name-field-description', + '.field--name-field-location-description', + ], + 'degewo.de': ['#section-description .c-copy'], + 'gesobau.de': ['.immoContent > .immoMainContent > p:first-of-type', '.immoContent > .immoMainContent > h3 + p'], + 'gewobag.de': ['.details-description > #objektbeschreibung + p', '.details-description > #lage + p'], + 'howoge.de': ['main .section > strong + p.readmore__wrap'], + 'stadtundland.de': [], + 'wbm.de': ['.openimmo-detail__intro-text', '.openimmo-detail__transport-connection-text'], +}; +const ALLOWED_HOSTS = new Set([...Object.keys(DETAIL_SELECTORS), 'inberlinwohnen.de']); +const ACTIVE_TEST_TIMEOUT_MS = 30_000; +const MAX_REDIRECTS = 5; + +let appliedBlackList = []; + +/** + * @param {unknown} value + * @param {string} label + * @returns {unknown} + */ +function findDetailValue(value, label) { + if (Array.isArray(value)) { + for (const entry of value) { + const found = findDetailValue(entry, label); + if (found != null) return found; + } + return null; + } + + if (value == null || typeof value !== 'object') return null; + if (value.label === label) return value.value; + + for (const entry of Object.values(value)) { + const found = findDetailValue(entry, label); + if (found != null) return found; + } + return null; +} + +/** + * @param {string|null|undefined} snapshot + * @returns {any} + */ +function parseItem(snapshot) { + try { + const item = JSON.parse(snapshot)?.data?.item; + return (Array.isArray(item) ? item[0] : item) || {}; + } catch { + return {}; + } +} + +/** + * Extract listing snapshots and pagination metadata from a server-rendered page. + * @param {string} html + * @returns {{listings: {id:string}[], totalPages:number, recognized:boolean, totalItems:number|null, itemsPerPage:number|null}} + */ +function parsePage(html) { + const $ = load(html); + const listings = []; + let totalPages = 1; + let recognized = false; + let totalItems = null; + let itemsPerPage = null; + + $('[wire\\:snapshot]').each((_, element) => { + const snapshot = $(element).attr('wire:snapshot'); + if (!snapshot) return; + + try { + const data = JSON.parse(snapshot)?.data; + const item = Array.isArray(data?.item) ? data.item[0] : data?.item; + if ( + item != null && + typeof item === 'object' && + firstScalarValue(item.id, item.objectId) != null && + typeof item.deeplink === 'string' + ) { + listings.push({ id: snapshot }); + recognized = true; + } + + const itemIds = Array.isArray(data?.itemIds?.[0]) ? data.itemIds[0] : data?.itemIds; + const pageSize = Number(data?.itemsPerPage); + if (Array.isArray(itemIds) && Number.isFinite(pageSize) && pageSize > 0) { + recognized = true; + const pageCount = Math.max(1, Math.ceil(itemIds.length / pageSize)); + if (pageCount >= totalPages) { + totalPages = pageCount; + totalItems = itemIds.length; + itemsPerPage = pageSize; + } + } + } catch { + // Other Livewire components may contain unrelated or incomplete snapshots. + } + }); + + return { listings, totalPages, recognized, totalItems, itemsPerPage }; +} + +/** + * Fetch every server-rendered result page so new jobs are not limited to page one. + * @param {string} url + * @param {any} browser + * @param {typeof puppeteerExtractor} [extractPage] + * @returns {Promise<{id:string}[]>} + */ +async function getListings(url, browser, extractPage = puppeteerExtractor) { + async function fetchPage(page) { + const pageUrl = new URL(url); + pageUrl.searchParams.set('page', String(page)); + const html = await extractPage(pageUrl.href, 'body', { browser, name: 'inberlinwohnen' }); + if (!html) throw new Error(`InBerlinWohnen page ${page} could not be loaded.`); + const parsed = parsePage(html); + if (!parsed.recognized) { + throw new Error(`InBerlinWohnen page ${page} did not contain the expected Livewire data.`); + } + if (parsed.totalItems != null && parsed.itemsPerPage != null) { + const expectedListings = Math.min( + parsed.itemsPerPage, + Math.max(0, parsed.totalItems - (page - 1) * parsed.itemsPerPage), + ); + if (parsed.listings.length < expectedListings) { + throw new Error( + `InBerlinWohnen page ${page} contained ${parsed.listings.length} of ${expectedListings} expected listings.`, + ); + } + } + return parsed; + } + + const firstPage = await fetchPage(1); + if (firstPage.totalPages > MAX_PAGES) { + throw new Error(`InBerlinWohnen returned ${firstPage.totalPages} pages, exceeding the safety limit.`); + } + + const listings = [...firstPage.listings]; + for (let page = 2; page <= firstPage.totalPages; page++) { + listings.push(...(await fetchPage(page)).listings); + } + const seen = new Set(); + return listings.filter((listing) => { + const item = parseItem(listing.id); + const key = firstScalarValue(item.id, item.objectId) ?? listing.id; + if (seen.has(String(key))) return false; + seen.add(String(key)); + return true; + }); +} + +/** + * @param {any} address + * @returns {string|null} + */ +function normalizeAddress(address) { + const value = Array.isArray(address) ? address[0] : address; + if (value == null || typeof value !== 'object') return null; + + const street = [value.street, value.number].filter(Boolean).join(' '); + const city = value.zipCode ? `${value.zipCode} Berlin` : null; + return [street, city, value.district].filter(Boolean).join(', ') || null; +} + +/** + * @param {...unknown} values + * @returns {string|number|null} + */ +function firstScalarValue(...values) { + return values.find((value) => typeof value === 'number' || (typeof value === 'string' && value.trim())) ?? null; +} + +/** + * @param {unknown} value + * @returns {string|null} + */ +function normalizeLink(value) { + if (typeof value !== 'string' || !value.trim()) return null; + try { + const url = new URL(value, BASE_URL); + const hostname = url.hostname.toLowerCase().replace(/^www\./, ''); + return (url.protocol === 'http:' || url.protocol === 'https:') && ALLOWED_HOSTS.has(hostname) ? url.href : null; + } catch { + return null; + } +} + +/** + * Enrich a listing from its partner detail page when a supported selector is available. + * @param {ParsedListing} listing + * @param {any} browser + * @param {typeof puppeteerExtractor} [extractDetails] + * @returns {Promise} + */ +async function fetchDetails(listing, browser, extractDetails = puppeteerExtractor) { + try { + const hostname = new URL(listing.link).hostname.toLowerCase().replace(/^www\./, ''); + const selectors = DETAIL_SELECTORS[hostname]; + if (!selectors?.length) return listing; + + const html = await extractDetails(listing.link, null, { + browser, + name: `inberlinwohnen_details_${hostname.replaceAll('.', '_')}`, + }); + if (!html) return listing; + + const $ = load(html); + const details = []; + for (const selector of selectors) { + $(selector).each((_, element) => { + const node = $(element).clone(); + node.find('script, style, noscript, button').remove(); + const text = node + .text() + .replace(/\s+/g, ' ') + .replace(/^(Beschreibung|Ausstattung|Lageinformationen?)\s*:?\s*/i, '') + .replace(/\s*mehr anzeigen\s*$/i, '') + .trim(); + if (text.length >= 20 && !details.includes(text)) details.push(text); + }); + } + if (!details.length) return listing; + + return { + ...listing, + description: [listing.description, ...details].filter(Boolean).join('\n\n'), + }; + } catch (error) { + logger.warn(`Could not fetch InBerlinWohnen detail page for listing '${listing.id}'.`, error?.message || error); + return listing; + } +} + +/** + * Follow aggregator redirects and only report definitive gone responses as inactive. + * @param {string} link + * @returns {Promise} + */ +async function isListingActive(link) { + try { + let currentUrl = normalizeLink(link); + if (!currentUrl) return -1; + const signal = AbortSignal.timeout(ACTIVE_TEST_TIMEOUT_MS); + + for (let redirectCount = 0; redirectCount <= MAX_REDIRECTS; redirectCount++) { + const response = await fetch(currentUrl, { redirect: 'manual', signal }); + if (response.status === 200) return 1; + if (response.status === 404 || response.status === 410) return 0; + if (response.status < 300 || response.status >= 400) return -1; + + const location = response.headers?.get('location'); + currentUrl = location ? normalizeLink(new URL(location, currentUrl).href) : null; + if (!currentUrl) return -1; + } + return -1; + } catch { + return -1; + } +} + +/** + * @param {any} o + * @returns {ParsedListing} + */ +function normalize(o) { + const item = parseItem(o.id); + const originalId = firstScalarValue(item.objectId, item.id); + const totalRent = firstScalarValue(findDetailValue(item.details, 'Gesamtmiete'), item.rentGross, item.rentNet); + const company = typeof item.company?.[0]?.name === 'string' ? item.company[0].name.trim() : null; + const wbs = findDetailValue(item.details, 'WBS'); + const description = [ + company ? `Anbieter: ${company}` : null, + item.rentNet != null ? `Kaltmiete: ${item.rentNet} €` : null, + item.extraCosts != null ? `Nebenkosten: ${item.extraCosts} €` : null, + totalRent != null ? `Gesamtmiete: ${totalRent} €` : null, + wbs ? `WBS: ${wbs}` : null, + item.occupationDate ? `Bezugsfertig ab: ${item.occupationDate}` : null, + ] + .filter(Boolean) + .join('\n'); + + return { + id: originalId == null ? null : buildHash(String(originalId)), + link: normalizeLink(item.deeplink), + title: typeof item.title === 'string' && item.title.trim() ? item.title.trim() : null, + price: extractNumber(totalRent), + size: extractNumber(firstScalarValue(item.area)), + rooms: extractNumber(firstScalarValue(item.rooms)), + address: normalizeAddress(item.address), + image: + typeof item.imagePath === 'string' && item.imagePath.trim() + ? new URL(`/img/${item.imagePath.replace(/^\/+/, '')}?q=90&fit=crop&fm=png&dpr=1`, BASE_URL).href + : null, + description: description || undefined, + }; +} + +/** + * @param {ParsedListing} o + * @returns {boolean} + */ +function applyBlacklist(o) { + return !isOneOf(o.title, appliedBlackList) && !isOneOf(o.description, appliedBlackList); +} + +/** @type {ProviderConfig} */ +const config = { + requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'], + url: null, + crawlContainer: '[wire\\:snapshot*=\'"item":\']', + sortByDateParam: null, + waitForSelector: 'body', + crawlFields: { + id: '@wire:snapshot', + }, + normalize, + filter: applyBlacklist, + activeTester: isListingActive, + fetchDetails, + getListings, +}; + +export const init = (sourceConfig, blacklist) => { + config.enabled = sourceConfig.enabled; + config.url = sourceConfig.url; + appliedBlackList = blacklist || []; +}; + +export const metaInformation = { + name: 'InBerlinWohnen', + baseUrl: `${BASE_URL}/`, + id: 'inberlinwohnen', +}; + +export { config }; diff --git a/lib/services/jobs/jobExecutionService.js b/lib/services/jobs/jobExecutionService.js index a32ca144..a36e6c24 100644 --- a/lib/services/jobs/jobExecutionService.js +++ b/lib/services/jobs/jobExecutionService.js @@ -186,7 +186,7 @@ export function initJobExecutionService({ providers, settings, intervalMs }) { browser = null; } - if (!browser && matchedProvider.config.getListings == null) { + if (!browser) { browser = await puppeteerExtractor.launchBrowser(matchedProvider.config.url, proxyUrl ? { proxyUrl } : {}); } diff --git a/lib/services/similarity-check/similarityCache.js b/lib/services/similarity-check/similarityCache.js index 964e6d4a..6c49d137 100644 --- a/lib/services/similarity-check/similarityCache.js +++ b/lib/services/similarity-check/similarityCache.js @@ -7,7 +7,7 @@ * Similarity cache * * Maintains an in-memory Set of content hashes to detect whether a listing - * (identified by a tuple of title, price and address) has been seen before. + * (identified by a tuple of job, title, price and address) has been seen before. * * Design notes: * - The cache is refreshed periodically from persistent storage. To avoid @@ -29,7 +29,7 @@ const reloadCycle = 60 * 60 * 1000; // every hour, refresh /** * Internal cache of content hashes for known listings. * - * Each entry is an SHA-256 hex digest produced by toHash(title, price, address). + * Each entry is an SHA-256 hex digest produced by toHash(jobId, title, price, address). * @type {Set} */ let cache = new Set(); @@ -55,7 +55,7 @@ export const initSimilarityCache = () => { const allEntries = getAllEntriesFromListings(); const newCache = new Set(); for (const entry of allEntries) { - newCache.add(toHash(entry?.title, entry?.price, entry?.address)); + newCache.add(toHash(entry?.job_id, entry?.title, entry?.price, entry?.address)); } // Atomic swap to avoid mutating the cache while it may be iterated elsewhere cache = newCache; @@ -64,18 +64,20 @@ export const initSimilarityCache = () => { /** * Check if a listing is already known and add it to the cache if not. * - * The listing is identified by the combination of its title, price and - * address. Null/undefined fields are ignored during hashing. Falsy-but-valid - * values (e.g., price 0) are preserved. + * The listing is identified by the combination of its job, title, price and + * address. This keeps cross-provider deduplication within a job without hiding + * the same listing from another job. Null/undefined fields are ignored during + * hashing. Falsy-but-valid values (e.g., price 0) are preserved. * * @param {Object} params - Listing fields + * @param {string} params.jobId - Job containing the listing * @param {string|undefined|null} params.title - The listing title * @param {string|undefined|null} params.address - The listing address * @param {number|string|undefined|null} params.price - The listing price * @returns {boolean} true if the entry already existed in the cache (duplicate), otherwise false */ -export const checkAndAddEntry = ({ title, address, price }) => { - const hash = toHash(title, price, address); +export const checkAndAddEntry = ({ jobId, title, address, price }) => { + const hash = toHash(jobId, title, price, address); if (cache.has(hash)) { return true; } @@ -106,25 +108,30 @@ export const checkAndAddEntry = ({ title, address, price }) => { * ignored, falsy-but-valid values like 0 preserved). * * @param {Object} params - Listing fields identifying the entry to evict. + * @param {string} params.jobId - Job containing the listing. * @param {string|undefined|null} params.title - The listing title. * @param {string|undefined|null} params.address - The listing address. * @param {number|string|undefined|null} params.price - The listing price. * @returns {boolean} true if an entry was removed, false if it was not present. */ -export const removeEntry = ({ title, address, price }) => { - return cache.delete(toHash(title, price, address)); +export const removeEntry = ({ jobId, title, address, price }) => { + return cache.delete(toHash(jobId, title, price, address)); }; /** - * Generate an SHA-256 hash from a list of input values. + * Generate an SHA-256 hash from a listing's similarity fields. + * Parenthesized address suffixes are removed to match persisted listing data. * Null or undefined values are ignored. Falsy but valid values like 0 are preserved. - * Non-string values are coerced to strings prior to hashing. * - * @param {...(string|number|null|undefined)} strings - Input values to hash + * @param {string|null|undefined} jobId + * @param {string|null|undefined} title + * @param {number|string|null|undefined} price + * @param {string|null|undefined} address * @returns {string} Hexadecimal hash */ -function toHash(...strings) { - const normalized = strings +function toHash(jobId, title, price, address) { + const normalizedAddress = typeof address === 'string' ? address.replace(/\s*\([^)]*\)/g, '') : address; + const normalized = [jobId, title, price, normalizedAddress] .filter((v) => v !== null && v !== undefined) .map((v) => (typeof v === 'string' ? v : String(v))); return crypto.createHash('sha256').update(normalized.join('|')).digest('hex'); diff --git a/lib/services/storage/listingsStorage.js b/lib/services/storage/listingsStorage.js index 564e82c4..fd1411e3 100755 --- a/lib/services/storage/listingsStorage.js +++ b/lib/services/storage/listingsStorage.js @@ -427,7 +427,7 @@ export const deleteListingsByJobId = (jobId, hardDelete = false) => { if (!jobId) return; if (hardDelete) { return hardDeleteAndEvict( - `SELECT title, address, price + `SELECT job_id, title, address, price, manually_deleted FROM listings WHERE job_id = @jobId`, `DELETE FROM listings @@ -451,7 +451,7 @@ export const deleteListingsByJobId = (jobId, hardDelete = false) => { * then deleted, then each is evicted from the cache. See * {@link module:similarityCache.removeEntry} for why eviction is required. * - * @param {string} selectSql - Query selecting `title, address, price` of the rows to delete. + * @param {string} selectSql - Query selecting the similarity fields and deletion status of the rows to delete. * @param {string} deleteSql - The DELETE statement removing the same rows. * @param {Object|Array} params - Bind parameters shared by both statements. * @returns {any} The result from SqliteConnection.execute for the DELETE. @@ -459,7 +459,9 @@ export const deleteListingsByJobId = (jobId, hardDelete = false) => { function hardDeleteAndEvict(selectSql, deleteSql, params) { const removed = SqliteConnection.query(selectSql, params); const result = SqliteConnection.execute(deleteSql, params); - removed.forEach((row) => removeEntry({ title: row.title, address: row.address, price: row.price })); + removed + .filter((row) => !row.manually_deleted) + .forEach((row) => removeEntry({ jobId: row.job_id, title: row.title, address: row.address, price: row.price })); return result; } @@ -476,7 +478,7 @@ export const deleteListingsById = (ids, hardDelete = false) => { const placeholders = ids.map(() => '?').join(','); if (hardDelete) { return hardDeleteAndEvict( - `SELECT title, address, price + `SELECT job_id, title, address, price, manually_deleted FROM listings WHERE id IN (${placeholders})`, `DELETE FROM listings @@ -590,13 +592,13 @@ export const getListingsForMap = ({ jobId, userId = null, isAdmin = false } = {} }; /** - * Return all listings with only the fields: title, address, and price. + * Return all listings with the job and similarity fields. * This is the single helper requested for simple consumers. * - * @returns {{title: string|null, address: string|null, price: number|null}[]} + * @returns {{job_id: string, title: string|null, address: string|null, price: number|null}[]} */ export const getAllEntriesFromListings = () => { - return SqliteConnection.query(`SELECT title, address, price FROM listings WHERE manually_deleted = 0`); + return SqliteConnection.query(`SELECT job_id, title, address, price FROM listings WHERE manually_deleted = 0`); }; /** diff --git a/lib/types/providerConfig.js b/lib/types/providerConfig.js index 7b6f77f4..12bfaf42 100644 --- a/lib/types/providerConfig.js +++ b/lib/types/providerConfig.js @@ -15,8 +15,8 @@ * @property {string} [crawlContainer] CSS selector for the container holding listing items. * @property {(raw: any) => ParsedListing} normalize Function to convert raw scraped data into a ParsedListing shape. * @property {(listing: ParsedListing) => boolean} filter Function to filter out unwanted listings. - * @property {(url: string, waitForSelector?: string) => Promise} [getListings] Optional override to fetch listings. - * @property {(listing:ParsedListing, browser:any)=>Promise} [providerConfig.fetchDetails] Optional per-listing detail enrichment. Called in parallel for each new listing after deduplication. Receives the shared browser instance. Must always resolve (never reject). + * @property {(url: string, browser?: any) => Promise} [getListings] Optional override to fetch listings. Receives the shared browser instance. + * @property {(listing:ParsedListing, browser:any)=>Promise} [providerConfig.fetchDetails] Optional per-listing detail enrichment. Called sequentially for each new listing after deduplication. Receives the shared browser instance. Must always resolve (never reject). * @property {Object} [puppeteerOptions] Puppeteer specific options. * @property {boolean} [enabled] Whether the provider is enabled. * @property {(url: string) => Promise | number} [activeTester] Function to check if a listing is still active. diff --git a/lib/types/similarityCache.js b/lib/types/similarityCache.js index fb426e44..1dd46b7e 100644 --- a/lib/types/similarityCache.js +++ b/lib/types/similarityCache.js @@ -5,7 +5,7 @@ /** * @typedef {Object} SimilarityCache - * @property {(params: { title?: string, address?: string, price?: number|string }) => boolean} checkAndAddEntry Checks if a listing is similar and adds it if not. + * @property {(params: { jobId: string, title?: string, address?: string, price?: number|string }) => boolean} checkAndAddEntry Checks if a listing is similar within a job and adds it if not. */ export {}; diff --git a/test/pipeline_filtering.test.js b/test/pipeline_filtering.test.js index 0891eda9..6b96afd8 100644 --- a/test/pipeline_filtering.test.js +++ b/test/pipeline_filtering.test.js @@ -3,7 +3,7 @@ * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause */ -import { afterEach, expect } from 'vitest'; +import { afterEach, expect, vi } from 'vitest'; import { mockFredy, sseEvents } from './utils.js'; import * as mockStore from './mocks/mockStore.js'; import { get as getLastNotification } from './mocks/mockNotification.js'; @@ -13,7 +13,7 @@ describe('Issue reproduction: listings filtered by similarity or area should be const Fredy = await mockFredy(); const mockSimilarityCache = { - checkAndAddEntry: () => true, // always similar + checkAndAddEntry: vi.fn(() => true), // always similar }; const providerConfig = { @@ -45,6 +45,38 @@ describe('Issue reproduction: listings filtered by similarity or area should be } expect(mockStore.deletedIds).toContain('1'); + expect(mockSimilarityCache.checkAndAddEntry).toHaveBeenCalledWith({ + jobId: 'test-job', + title: 'test', + address: 'addr', + price: '100', + }); + }); + + it('should pass the shared browser to a custom getListings implementation', async () => { + const Fredy = await mockFredy(); + const browser = { connected: true }; + const getListings = vi.fn().mockResolvedValue([]); + const providerConfig = { + url: 'http://example.com', + getListings, + normalize: (listing) => listing, + filter: () => true, + crawlFields: {}, + requiredFieldNames: [], + }; + const mockedJob = { + id: 'custom-get-listings-browser', + notificationAdapter: null, + specFilter: null, + spatialFilter: null, + }; + + const fredy = new Fredy(providerConfig, mockedJob, 'custom-provider', {}, browser); + await fredy.execute(); + + expect(getListings).toHaveBeenCalledWith('http://example.com', browser); + expect(getListings.mock.contexts[0]).toBe(fredy); }); it('should call deleteListingsById when listings are filtered by area', async () => { diff --git a/test/provider/inberlinwohnen.test.js b/test/provider/inberlinwohnen.test.js new file mode 100644 index 00000000..999ac84b --- /dev/null +++ b/test/provider/inberlinwohnen.test.js @@ -0,0 +1,331 @@ +/* + * Copyright (c) 2026 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +import { afterAll, afterEach, beforeAll, expect, vi } from 'vitest'; +// Import utils.js before puppeteerExtractor so its offline vi.mock (which stubs +// launchBrowser) is registered first; otherwise offline runs launch a real +// browser and fail when the CloakBrowser binary isn't cached (e.g. in CI). +import { mockFredy, providerConfig } from '../utils.js'; +import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js'; +import { closeBrowser, launchBrowser } from '../../lib/services/extractor/puppeteerExtractor.js'; +import { get } from '../mocks/mockNotification.js'; +import * as provider from '../../lib/provider/inberlinwohnen.js'; +import { buildHash } from '../../lib/utils.js'; + +const offlineIt = process.env.TEST_MODE === 'offline' ? it : it.skip; +const liveIt = process.env.TEST_MODE === 'offline' ? it.skip : it; + +const snapshot = (item, tuple = true) => ({ + id: JSON.stringify({ data: { item: tuple ? [item, { s: 'arr' }] : item } }), +}); + +describe('#inberlinwohnen testsuite()', () => { + let browser; + let liveListings = []; + + provider.init(providerConfig.inberlinwohnen, []); + beforeAll(async () => { + browser = await launchBrowser(providerConfig.inberlinwohnen.url); + }); + afterAll(async () => { + await closeBrowser(browser); + }); + afterEach(() => { + vi.clearAllMocks(); + vi.restoreAllMocks(); + }); + + it('should parse listings from the Livewire snapshot', async () => { + const Fredy = await mockFredy(); + const mockedJob = { + id: 'inberlinwohnen', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache, browser); + + const listings = await fredy.execute(); + if (process.env.TEST_MODE !== 'offline') liveListings = listings; + + expect(listings.length).toBeGreaterThan(0); + const notification = get(); + expect(notification.serviceName).toBe('inberlinwohnen'); + expect(notification.payload.length).toBeGreaterThan(0); + notification.payload.forEach((listing) => { + expect(listing.id).toBeTypeOf('string'); + expect(listing.title).toBeTypeOf('string'); + expect(listing.title).not.toBe(''); + expect(listing.link).toMatch(/^https:\/\//); + expect(listing.price).toMatch(/ €$/); + expect(listing.size).toMatch(/ m²$/); + expect(listing.rooms).toMatch(/ Zimmer$/); + expect(listing.address).toContain('Berlin'); + if (listing.image) expect(listing.image).toContain('https://inberlinwohnen.de/img/images/apartments/'); + expect(listing.description).toContain('Gesamtmiete:'); + }); + expect(notification.payload.some((listing) => listing.image)).toBe(true); + + if (process.env.TEST_MODE === 'offline') { + expect(notification.payload).toHaveLength(1); + expect(notification.payload[0]).toMatchObject({ + title: 'Wohnung perfekt für Altbaufans', + link: 'https://www.degewo.de/de/properties/W1100-41909-0008-0303.html', + price: '600.37 €', + size: '51.04 m²', + rooms: '2 Zimmer', + address: 'Firlstraße 31, 12459 Berlin, Treptow-Köpenick', + image: 'https://inberlinwohnen.de/img/images/apartments/sample.webp?q=90&fit=crop&fm=png&dpr=1', + }); + expect(notification.payload[0].description).toContain('WBS: unbekannt'); + } + }); + + liveIt('should enrich available live partner detail pages', async () => { + const detailHosts = new Set(['berlinovo.de', 'degewo.de', 'gesobau.de', 'gewobag.de', 'howoge.de', 'wbm.de']); + const samples = new Map(); + for (const listing of liveListings) { + const hostname = new URL(listing.link).hostname.replace(/^www\./, ''); + if (detailHosts.has(hostname) && !samples.has(hostname)) samples.set(hostname, listing); + } + + expect([...samples.keys()].sort()).toEqual([...detailHosts].sort()); + for (const listing of samples.values()) { + const enriched = await provider.config.fetchDetails(listing, browser); + expect(enriched.description.length).toBeGreaterThan(listing.description.length); + } + }); + + offlineIt('should fetch every server-rendered result page', async () => { + const item = (id) => + `
`; + const page = (items, itemIds = null) => ` + ${ + itemIds == null + ? '' + : `
` + } + ${items.map(item).join('')} + `; + const extractorMock = vi + .fn() + .mockResolvedValueOnce(page([1, 2], [1, 2, 3, 4, 5])) + .mockResolvedValueOnce(page([3, 4])) + .mockResolvedValueOnce(page([5])); + + const listings = await provider.config.getListings( + 'https://inberlinwohnen.de/wohnungsfinder/?district=mitte&page=9', + browser, + extractorMock, + ); + + expect(listings).toHaveLength(5); + expect(listings.map((listing) => JSON.parse(listing.id).data.item[0].id)).toEqual([1, 2, 3, 4, 5]); + expect(extractorMock).toHaveBeenNthCalledWith( + 1, + 'https://inberlinwohnen.de/wohnungsfinder/?district=mitte&page=1', + 'body', + { + browser, + name: 'inberlinwohnen', + }, + ); + expect(extractorMock).toHaveBeenNthCalledWith( + 2, + 'https://inberlinwohnen.de/wohnungsfinder/?district=mitte&page=2', + 'body', + { browser, name: 'inberlinwohnen' }, + ); + expect(extractorMock).toHaveBeenNthCalledWith( + 3, + 'https://inberlinwohnen.de/wohnungsfinder/?district=mitte&page=3', + 'body', + { browser, name: 'inberlinwohnen' }, + ); + }); + + offlineIt('should reject pages without recognizable Livewire search data', async () => { + const extractorMock = vi.fn().mockResolvedValue( + `
+
`, + ); + + await expect( + provider.config.getListings(providerConfig.inberlinwohnen.url, browser, extractorMock), + ).rejects.toThrow('contained 0 of 1 expected listings'); + }); + + offlineIt('should reject a later page that omits expected listings', async () => { + const listing = `
`; + const pagination = `
`; + const extractorMock = vi.fn().mockResolvedValueOnce(`${pagination}${listing}`).mockResolvedValueOnce(pagination); + + await expect( + provider.config.getListings(providerConfig.inberlinwohnen.url, browser, extractorMock), + ).rejects.toThrow('page 2 contained 0 of 1 expected listings'); + }); + + offlineIt('should reject browser failures and excessive pagination', async () => { + await expect( + provider.config.getListings(providerConfig.inberlinwohnen.url, browser, vi.fn().mockResolvedValue(null)), + ).rejects.toThrow('could not be loaded'); + + const pagination = ` +
+
`; + await expect( + provider.config.getListings(providerConfig.inberlinwohnen.url, browser, vi.fn().mockResolvedValue(pagination)), + ).rejects.toThrow('exceeding the safety limit'); + }); + + offlineIt('should accept a recognized search with no results', async () => { + const emptySearch = `
`; + + await expect( + provider.config.getListings(providerConfig.inberlinwohnen.url, browser, vi.fn().mockResolvedValue(emptySearch)), + ).resolves.toEqual([]); + }); + + offlineIt('should deduplicate listings repeated across page boundaries', async () => { + const item = (id) => + `
`; + const page = (ids, itemIds = null) => ` + ${itemIds ? `
` : ''} + ${ids.map(item).join('')} + `; + const extractorMock = vi + .fn() + .mockResolvedValueOnce(page([1, 2], [1, 2, 3])) + .mockResolvedValueOnce(page([2, 3])); + + const listings = await provider.config.getListings(providerConfig.inberlinwohnen.url, browser, extractorMock); + + expect(listings.map((listing) => JSON.parse(listing.id).data.item[0].id)).toEqual([1, 2, 3]); + }); + + offlineIt.each([ + ['berlinovo.de', '
Helle Wohnung mit Einbauküche und Balkon.
'], + [ + 'www.degewo.de', + '
Ruhige Wohnung im obersten Geschoss.
', + ], + [ + 'www.gesobau.de', + '

Wohnung mit Serviceangeboten für Seniorinnen und Senioren.

', + ], + [ + 'www.gewobag.de', + '

Neubauwohnung mit Balkon und Fußbodenheizung.

', + ], + [ + 'www.howoge.de', + '
Beschreibung

Drei Zimmer mit Balkon und modernem Bad.

', + ], + ['www.wbm.de', '
Gut geschnittene Wohnung mit hellen Wohnräumen.
'], + ])('should enrich detail descriptions from %s', async (hostname, html) => { + const extractorMock = vi.fn().mockResolvedValue(html); + const listing = { + id: 'listing-1', + link: `https://${hostname}/listing/1`, + description: 'Gesamtmiete: 600 €', + }; + + const enriched = await provider.config.fetchDetails(listing, browser, extractorMock); + + expect(enriched.description).toContain('Gesamtmiete: 600 €'); + expect(enriched.description).not.toBe(listing.description); + expect(extractorMock).toHaveBeenCalledWith(listing.link, null, expect.objectContaining({ browser })); + }); + + offlineIt('should preserve listings when detail enrichment is unsupported or unavailable', async () => { + const unsupported = { id: '1', link: 'https://example.com/listing/1', description: 'Original' }; + expect(await provider.config.fetchDetails(unsupported, browser)).toBe(unsupported); + + const supported = { ...unsupported, link: 'https://www.degewo.de/listing/1' }; + const extractorMock = vi.fn().mockResolvedValue(null); + expect(await provider.config.fetchDetails(supported, browser, extractorMock)).toBe(supported); + + const stadtUndLand = { ...unsupported, link: 'https://stadtundland.de/wohnungssuche/1' }; + expect(await provider.config.fetchDetails(stadtUndLand, browser, extractorMock)).toBe(stadtUndLand); + }); + + it('should support object snapshots and rent fallbacks', () => { + const baseItem = { + id: 19252, + title: 'Fallback listing', + deeplink: '/wohnungsfinder/fallback', + rooms: '2,0', + area: '51,04', + }; + + const objectListing = provider.config.normalize(snapshot({ ...baseItem, rentGross: '503,91' }, false)); + const netRentListing = provider.config.normalize(snapshot({ ...baseItem, rentNet: '428,38' })); + + expect(objectListing.price).toBe(503.91); + expect(objectListing.link).toBe('https://inberlinwohnen.de/wohnungsfinder/fallback'); + expect(netRentListing.price).toBe(428.38); + expect(netRentListing.id).toBe(objectListing.id); + expect(provider.config.normalize(snapshot({ ...baseItem, deeplink: 'http://%' })).link).toBeNull(); + expect(provider.config.normalize(snapshot({ ...baseItem, deeplink: 'javascript:alert(1)' })).link).toBeNull(); + + const stablePartnerId = provider.config.normalize(snapshot({ ...baseItem, id: 100, objectId: 'partner-42' })); + expect(stablePartnerId.id).toBe(buildHash('partner-42')); + }); + + it('should apply blacklist terms to titles and descriptions', () => { + provider.init(providerConfig.inberlinwohnen, ['wbs']); + + expect(provider.config.filter({ title: 'Wohnung mit WBS', description: '' })).toBe(false); + expect(provider.config.filter({ title: 'Wohnung', description: 'WBS erforderlich' })).toBe(false); + expect(provider.config.filter({ title: 'Wohnung', description: 'Bezugsfertig' })).toBe(true); + + provider.init(providerConfig.inberlinwohnen, []); + }); + + it('should follow safe redirects when checking if a listing is active', async () => { + const originalFetch = globalThis.fetch; + const fetchMock = vi + .fn() + .mockResolvedValueOnce({ status: 302, headers: new Headers({ location: '/properties/new.html' }) }) + .mockResolvedValueOnce({ status: 200 }) + .mockResolvedValueOnce({ status: 404 }) + .mockResolvedValueOnce({ status: 410 }) + .mockResolvedValueOnce({ status: 503 }) + .mockRejectedValueOnce(new Error('network failure')); + globalThis.fetch = fetchMock; + + try { + await expect(provider.config.activeTester('https://www.degewo.de/redirect')).resolves.toBe(1); + await expect(provider.config.activeTester('https://www.degewo.de/gone')).resolves.toBe(0); + await expect(provider.config.activeTester('https://www.degewo.de/removed')).resolves.toBe(0); + await expect(provider.config.activeTester('https://www.degewo.de/unavailable')).resolves.toBe(-1); + await expect(provider.config.activeTester('https://www.degewo.de/network-error')).resolves.toBe(-1); + await expect(provider.config.activeTester('http://127.0.0.1/private')).resolves.toBe(-1); + expect(fetchMock).toHaveBeenNthCalledWith( + 1, + 'https://www.degewo.de/redirect', + expect.objectContaining({ redirect: 'manual', signal: expect.any(AbortSignal) }), + ); + expect(fetchMock).toHaveBeenNthCalledWith( + 2, + 'https://www.degewo.de/properties/new.html', + expect.objectContaining({ redirect: 'manual', signal: expect.any(AbortSignal) }), + ); + expect(fetchMock).toHaveBeenCalledTimes(6); + } finally { + globalThis.fetch = originalFetch; + } + }); +}); diff --git a/test/provider/testProvider.json b/test/provider/testProvider.json index 897b64a2..4fbf5f6f 100644 --- a/test/provider/testProvider.json +++ b/test/provider/testProvider.json @@ -20,6 +20,10 @@ "url": "https://immo.swp.de/suchergebnisse?l=M%C3%BCnchen&r=0km&_multiselect_r=0km&ut=private&t=apartment%3Arental&a=de.muenchen&pf=&pt=&rf=0&rt=0&sf=50&st=&yf=&yt=&ff=&ft=&s=most_recently_updated_first&pa=&o=&ad=&u=", "enabled": true }, + "inberlinwohnen": { + "url": "https://inberlinwohnen.de/wohnungsfinder/", + "enabled": true + }, "kleinanzeigen": { "url": "https://www.kleinanzeigen.de/s-immobilien/duesseldorf/anzeige:angebote/wohnung/k0c195l2068r5", "enabled": true diff --git a/test/services/jobs/jobExecutionService.test.js b/test/services/jobs/jobExecutionService.test.js index 0f8b5db6..59b56f1f 100644 --- a/test/services/jobs/jobExecutionService.test.js +++ b/test/services/jobs/jobExecutionService.test.js @@ -23,6 +23,8 @@ describe('services/jobs/jobExecutionService', () => { const utilsPath = root + '/lib/utils.js'; const loggerPath = root + '/lib/services/logger.js'; const notifyPath = root + '/lib/notification/notify.js'; + const pipelinePath = root + '/lib/FredyPipelineExecutioner.js'; + const puppeteerPath = root + '/lib/services/extractor/puppeteerExtractor.js'; vi.resetModules(); vi.doMock(busPath, () => ({ bus })); @@ -50,27 +52,55 @@ describe('services/jobs/jobExecutionService', () => { return { default: m }; }); vi.doMock(notifyPath, () => ({ send: async () => [] })); + vi.doMock(puppeteerPath, () => ({ + launchBrowser: async (...args) => { + calls.launchBrowser.push(args); + return state.browser; + }, + closeBrowser: async (browser) => { + calls.closeBrowser.push(browser); + }, + })); + vi.doMock(pipelinePath, () => ({ + default: class { + constructor(config, job, providerId, similarityCache, browser) { + calls.pipeline.push({ config, job, providerId, similarityCache, browser }); + } + + async execute() {} + }, + })); vi.doMock(root + '/lib/services/jobs/run-state.js', () => ({ isRunning: () => false, markRunning: (id) => { calls.markRunning.push(id); return true; }, - markFinished: () => {}, + markFinished: (id) => calls.markFinished.push(id), })); const mod = await import(svcPath); - mod.initJobExecutionService({ providers: [], settings: { demoMode: false }, intervalMs: 0 }); + mod.initJobExecutionService({ providers: state.providers, settings: { demoMode: false }, intervalMs: 0 }); return mod; } beforeEach(() => { bus = new EventEmitter(); - calls = { sent: [], markRunning: [], lastRunUpdates: [] }; + calls = { + sent: [], + markRunning: [], + markFinished: [], + lastRunUpdates: [], + launchBrowser: [], + closeBrowser: [], + pipeline: [], + }; state = { jobsById: {}, jobsList: [], users: [], + providers: [], + browser: { connected: true }, }; }); @@ -139,4 +169,37 @@ describe('services/jobs/jobExecutionService', () => { expect(update.timestamp).toBeGreaterThanOrEqual(before); expect(update.timestamp).toBeLessThanOrEqual(after); }); + + it('launches and reuses a single shared browser across all providers in a job', async () => { + const provider = (id, config) => ({ + metaInformation: { id }, + config, + init: vi.fn(), + }); + state.providers = [ + provider('api-provider', { url: 'https://api.example/', getListings: vi.fn() }), + provider('browser-provider', { + url: 'https://browser.example/', + getListings: vi.fn(), + }), + provider('browser-provider-2', { + url: 'https://browser-2.example/', + getListings: vi.fn(), + }), + ]; + state.jobsById.j1 = { + id: 'j1', + enabled: true, + userId: 'u1', + provider: state.providers.map(({ metaInformation }) => ({ id: metaInformation.id })), + }; + + await initService(); + bus.emit('jobs:runOne', { jobId: 'j1' }); + await vi.waitFor(() => expect(calls.markFinished).toEqual(['j1'])); + + expect(calls.launchBrowser).toEqual([['https://api.example/', {}]]); + expect(calls.pipeline.map(({ browser }) => browser)).toEqual([state.browser, state.browser, state.browser]); + expect(calls.closeBrowser).toEqual([state.browser]); + }); }); diff --git a/test/similarity/similarityCache.test.js b/test/similarity/similarityCache.test.js index 639471a3..af613baf 100644 --- a/test/similarity/similarityCache.test.js +++ b/test/similarity/similarityCache.test.js @@ -17,29 +17,30 @@ async function loadModuleWith({ entries = [] } = {}) { describe('similarityCache', () => { it('initSimilarityCache builds cache from storage and enables duplicate detection', async () => { const entries = [ - { title: 'A', price: 1000, address: 'Main 1' }, - { title: 'B', price: 0, address: 'Zero St' }, + { job_id: 'job-1', title: 'A', price: 1000, address: 'Main 1' }, + { job_id: 'job-1', title: 'B', price: 0, address: 'Zero St' }, ]; const { initSimilarityCache, checkAndAddEntry } = await loadModuleWith({ entries }); // Initially, duplicates should not be detected for new data - expect(checkAndAddEntry({ title: 'X', price: 200, address: 'Y' })).toBe(false); + expect(checkAndAddEntry({ jobId: 'job-1', title: 'X', price: 200, address: 'Y' })).toBe(false); // Now initialize from storage initSimilarityCache(); // Exact duplicates should be detected - expect(checkAndAddEntry({ title: 'A', price: 1000, address: 'Main 1' })).toBe(true); + expect(checkAndAddEntry({ jobId: 'job-1', title: 'A', price: 1000, address: 'Main 1' })).toBe(true); + expect(checkAndAddEntry({ jobId: 'job-2', title: 'A', price: 1000, address: 'Main 1' })).toBe(false); // Ensure falsy-but-valid price 0 is preserved by hashing and detected as duplicate - expect(checkAndAddEntry({ title: 'B', price: 0, address: 'Zero St' })).toBe(true); + expect(checkAndAddEntry({ jobId: 'job-1', title: 'B', price: 0, address: 'Zero St' })).toBe(true); }); it('checkAndAddEntry returns false for new entry then true for duplicate on second call', async () => { const { checkAndAddEntry } = await loadModuleWith(); - const first = checkAndAddEntry({ title: 'C', price: 300, address: 'Road 3' }); - const second = checkAndAddEntry({ title: 'C', price: 300, address: 'Road 3' }); + const first = checkAndAddEntry({ jobId: 'job-1', title: 'C', price: 300, address: 'Road 3' }); + const second = checkAndAddEntry({ jobId: 'job-1', title: 'C', price: 300, address: 'Road 3' }); expect(first).toBe(false); expect(second).toBe(true); @@ -49,16 +50,16 @@ describe('similarityCache', () => { const { checkAndAddEntry } = await loadModuleWith(); // Add baseline (null address ignored) - const add1 = checkAndAddEntry({ title: 'T', price: 1, address: null }); + const add1 = checkAndAddEntry({ jobId: 'job-1', title: 'T', price: 1, address: null }); expect(add1).toBe(false); // Duplicate with undefined address should match - const dup = checkAndAddEntry({ title: 'T', price: 1, address: undefined }); + const dup = checkAndAddEntry({ jobId: 'job-1', title: 'T', price: 1, address: undefined }); expect(dup).toBe(true); // Now test that price 0 is preserved (not filtered out) - const addZero = checkAndAddEntry({ title: 'Z', price: 0, address: 'Zero' }); + const addZero = checkAndAddEntry({ jobId: 'job-1', title: 'Z', price: 0, address: 'Zero' }); expect(addZero).toBe(false); - const dupZero = checkAndAddEntry({ title: 'Z', price: 0, address: 'Zero' }); + const dupZero = checkAndAddEntry({ jobId: 'job-1', title: 'Z', price: 0, address: 'Zero' }); expect(dupZero).toBe(true); }); @@ -66,30 +67,58 @@ describe('similarityCache', () => { const { checkAndAddEntry, removeEntry } = await loadModuleWith(); // Seed the cache with an entry - expect(checkAndAddEntry({ title: 'A', price: 1000, address: 'Main 1' })).toBe(false); - expect(checkAndAddEntry({ title: 'A', price: 1000, address: 'Main 1' })).toBe(true); + expect(checkAndAddEntry({ jobId: 'job-1', title: 'A', price: 1000, address: 'Main 1' })).toBe(false); + expect(checkAndAddEntry({ jobId: 'job-1', title: 'A', price: 1000, address: 'Main 1' })).toBe(true); // Evict it - expect(removeEntry({ title: 'A', price: 1000, address: 'Main 1' })).toBe(true); + expect(removeEntry({ jobId: 'job-1', title: 'A', price: 1000, address: 'Main 1' })).toBe(true); // After eviction it must be treated as new again (this is the hard-delete fix) - expect(checkAndAddEntry({ title: 'A', price: 1000, address: 'Main 1' })).toBe(false); + expect(checkAndAddEntry({ jobId: 'job-1', title: 'A', price: 1000, address: 'Main 1' })).toBe(false); }); it('removeEntry returns false when the entry is not present', async () => { const { removeEntry } = await loadModuleWith(); - expect(removeEntry({ title: 'Nope', price: 1, address: 'Nowhere' })).toBe(false); + expect(removeEntry({ jobId: 'job-1', title: 'Nope', price: 1, address: 'Nowhere' })).toBe(false); }); it('removeEntry uses the same hashing rules (null/undefined ignored, 0 preserved)', async () => { const { checkAndAddEntry, removeEntry } = await loadModuleWith(); // Seed with a null address and price 0 - expect(checkAndAddEntry({ title: 'Z', price: 0, address: null })).toBe(false); + expect(checkAndAddEntry({ jobId: 'job-1', title: 'Z', price: 0, address: null })).toBe(false); // Removing with undefined address (same hash) should evict it - expect(removeEntry({ title: 'Z', price: 0, address: undefined })).toBe(true); - expect(checkAndAddEntry({ title: 'Z', price: 0, address: null })).toBe(false); + expect(removeEntry({ jobId: 'job-1', title: 'Z', price: 0, address: undefined })).toBe(true); + expect(checkAndAddEntry({ jobId: 'job-1', title: 'Z', price: 0, address: null })).toBe(false); + }); + + it('keeps similarity detection isolated between jobs', async () => { + const { checkAndAddEntry } = await loadModuleWith(); + const listing = { title: 'A', price: 1000, address: 'Main 1' }; + + expect(checkAndAddEntry({ jobId: 'job-1', ...listing })).toBe(false); + expect(checkAndAddEntry({ jobId: 'job-1', ...listing })).toBe(true); + expect(checkAndAddEntry({ jobId: 'job-2', ...listing })).toBe(false); + }); + + it('normalizes parenthesized address suffixes consistently with storage', async () => { + const { checkAndAddEntry } = await loadModuleWith(); + + expect(checkAndAddEntry({ jobId: 'job-1', title: 'A', price: 1000, address: 'Main 1 (Mitte)' })).toBe(false); + expect(checkAndAddEntry({ jobId: 'job-1', title: 'A', price: 1000, address: 'Main 1' })).toBe(true); + }); + + it('removes only the matching job entry', async () => { + const { checkAndAddEntry, removeEntry } = await loadModuleWith(); + const listing = { title: 'A', price: 1000, address: 'Main 1' }; + + expect(checkAndAddEntry({ jobId: 'job-1', ...listing })).toBe(false); + expect(checkAndAddEntry({ jobId: 'job-2', ...listing })).toBe(false); + expect(removeEntry({ jobId: 'job-1', ...listing })).toBe(true); + + expect(checkAndAddEntry({ jobId: 'job-1', ...listing })).toBe(false); + expect(checkAndAddEntry({ jobId: 'job-2', ...listing })).toBe(true); }); }); diff --git a/test/storage/deleteListings.test.js b/test/storage/deleteListings.test.js index 7b4cf9d7..e9aa712d 100644 --- a/test/storage/deleteListings.test.js +++ b/test/storage/deleteListings.test.js @@ -50,12 +50,13 @@ describe('listingsStorage hard delete evicts the similarity cache', () => { describe('deleteListingsByJobId', () => { it('hard delete fetches affected rows, DELETEs them and evicts each from the cache', () => { sqliteMock.__queryHandler = () => [ - { title: 'A', address: 'Main 1', price: 1000 }, - { title: 'B', address: 'Zero St', price: 0 }, + { job_id: 'job-1', title: 'A', address: 'Main 1', price: 1000 }, + { job_id: 'job-1', title: 'B', address: 'Zero St', price: 0 }, ]; listingsStorage.deleteListingsByJobId('job-1', true); + expect(calls.query[0].sql).toMatch(/SELECT job_id, title, address, price, manually_deleted/); // A DELETE (not a soft-delete UPDATE) must run expect(calls.execute).toHaveLength(1); expect(calls.execute[0].sql).toMatch(/DELETE FROM listings/); @@ -63,8 +64,8 @@ describe('listingsStorage hard delete evicts the similarity cache', () => { // Each removed row must be evicted from the similarity cache expect(removeEntry).toHaveBeenCalledTimes(2); - expect(removeEntry).toHaveBeenCalledWith({ title: 'A', address: 'Main 1', price: 1000 }); - expect(removeEntry).toHaveBeenCalledWith({ title: 'B', address: 'Zero St', price: 0 }); + expect(removeEntry).toHaveBeenCalledWith({ jobId: 'job-1', title: 'A', address: 'Main 1', price: 1000 }); + expect(removeEntry).toHaveBeenCalledWith({ jobId: 'job-1', title: 'B', address: 'Zero St', price: 0 }); }); it('soft delete marks rows and does NOT touch the similarity cache', () => { @@ -75,6 +76,17 @@ describe('listingsStorage hard delete evicts the similarity cache', () => { expect(removeEntry).not.toHaveBeenCalled(); }); + it('does not evict cache entries for already hidden duplicates', () => { + sqliteMock.__queryHandler = () => [ + { job_id: 'job-1', title: 'A', address: 'Main 1', price: 1000, manually_deleted: 1 }, + ]; + + listingsStorage.deleteListingsByJobId('job-1', true); + + expect(calls.execute[0].sql).toMatch(/DELETE FROM listings/); + expect(removeEntry).not.toHaveBeenCalled(); + }); + it('is a no-op without a jobId', () => { listingsStorage.deleteListingsByJobId(undefined, true); expect(calls.execute).toHaveLength(0); @@ -84,7 +96,7 @@ describe('listingsStorage hard delete evicts the similarity cache', () => { describe('deleteListingsById', () => { it('hard delete fetches affected rows, DELETEs them and evicts each from the cache', () => { - sqliteMock.__queryHandler = () => [{ title: 'C', address: 'Road 3', price: 300 }]; + sqliteMock.__queryHandler = () => [{ job_id: 'job-1', title: 'C', address: 'Road 3', price: 300 }]; listingsStorage.deleteListingsById(['id-1', 'id-2'], true); @@ -93,7 +105,7 @@ describe('listingsStorage hard delete evicts the similarity cache', () => { expect(calls.execute[0].sql).not.toMatch(/manually_deleted/); expect(removeEntry).toHaveBeenCalledTimes(1); - expect(removeEntry).toHaveBeenCalledWith({ title: 'C', address: 'Road 3', price: 300 }); + expect(removeEntry).toHaveBeenCalledWith({ jobId: 'job-1', title: 'C', address: 'Road 3', price: 300 }); }); it('soft delete marks rows and does NOT touch the similarity cache', () => { diff --git a/test/testFixtures/inberlinwohnen.html b/test/testFixtures/inberlinwohnen.html new file mode 100644 index 00000000..e8504727 --- /dev/null +++ b/test/testFixtures/inberlinwohnen.html @@ -0,0 +1,9 @@ + + + +
+
+ +