From 5882540cf8f35de1ff3d300514fc695b86e9481d Mon Sep 17 00:00:00 2001 From: gitea Date: Fri, 21 Aug 2026 21:40:16 +0000 Subject: [PATCH] fix(scraper): perfect review dialog opening, pagination, and multi-page review extraction --- src/index.js | 276 ++++++++++++++++++--------------------------------- 1 file changed, 94 insertions(+), 182 deletions(-) diff --git a/src/index.js b/src/index.js index bbe898d..237de67 100644 --- a/src/index.js +++ b/src/index.js @@ -23,7 +23,10 @@ const WESTERN_COUNTRIES = [ 'United Kingdom', 'UK', 'Niederlande', 'Netherlands', 'Frankreich', 'France', 'Spanien', 'Spain', 'Italien', 'Italy', 'Kanada', 'Canada', 'Australien', 'Australia', 'Belgien', 'Belgium', 'Schweden', 'Sweden', 'Norwegen', 'Norway', 'Dänemark', 'Denmark', - 'Finnland', 'Finland', 'Irland', 'Ireland', 'Luxemburg', 'Luxembourg', 'Indien', 'India' + 'Finnland', 'Finland', 'Irland', 'Ireland', 'Luxemburg', 'Luxembourg', 'Indien', 'India', + 'Singapur', 'Singapore', 'Malaysia', 'Sri Lanka', 'Neuseeland', 'New Zealand', + 'Südafrika', 'South Africa', 'Vereinigte Arabische Emirate', 'UAE', 'Dubai', + 'Polen', 'Poland', 'Tschechien', 'Czech', 'Japan', 'Korea' ]; // ========================================== @@ -163,8 +166,8 @@ async function safeEvaluate(page, fn, fallback = null, arg = undefined) { // BULLMQ WORKER: SCRAPING & REVIEWS // ========================================== const worker = new Worker('scrape-jobs', async (job) => { - const { url, webhookUrl, appWebhookUrl, hotelId, debug } = job.data; - const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http')); + const { url, webhookUrl, appWebhookUrl, hotelId, debug, mode } = job.data; + const isReviewScan = Boolean((webhookUrl && webhookUrl.startsWith('http')) || mode === 'reviews' || mode === 'full'); const isDebugMode = Boolean(debug || DEBUG_HTML); console.log(`\n==================================================`); @@ -187,29 +190,21 @@ const worker = new Worker('scrape-jobs', async (job) => { args: [ '--no-sandbox', '--disable-setuid-sandbox', + '--disable-dev-shm-usage', '--disable-blink-features=AutomationControlled', - '--disable-infobars' + '--window-size=1920,1080' ] }); const context = await browser.newContext({ - userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', - viewport: { width: 1366, height: 900 }, + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', + viewport: { width: 1920, height: 1080 }, locale: 'de-DE', - timezoneId: 'Europe/Berlin', - extraHTTPHeaders: { - 'Accept-Language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7', - 'Upgrade-Insecure-Requests': '1', - 'Sec-Ch-Ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"', - 'Sec-Ch-Ua-Mobile': '?0', - 'Sec-Ch-Ua-Platform': '"Windows"' - } + timezoneId: 'Europe/Berlin' }); - const page = await context.newPage(); - // Verhindert Erkennung über navigator.webdriver und emuliert Standard-Browser-Eigenschaften - await page.addInitScript(() => { + await context.addInitScript(() => { Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); window.chrome = { runtime: {}, @@ -225,6 +220,8 @@ const worker = new Worker('scrape-jobs', async (job) => { }); }); + const page = await context.newPage(); + let targetUrl = cleanBookingUrl(url); console.log(`[Job ${job.id}] 🌐 Bereinigte Ziel-URL: ${targetUrl}`); @@ -289,20 +286,12 @@ const worker = new Worker('scrape-jobs', async (job) => { // Cookie-Banner & störende Overlays für alle Scans sofort schließen try { - const dismissSelectors = [ - '#onetrust-accept-btn-handler', - 'button#onetrust-reject-all-handler', - 'button[data-testid="selection-modal-close-btn"]', - 'button[aria-label="Schließen"]', - 'button[aria-label="Close"]', - 'button.modal-mask-closeBtn' - ]; - for (const btnSel of dismissSelectors) { - const btn = await page.$(btnSel); - if (btn) { - await btn.click().catch(() => {}); - await page.waitForTimeout(300); - } + const dismissBtns = await page.$$( + 'button[aria-label*="schließen"], button[aria-label*="Schließen"], button[aria-label*="Close"], button[aria-label*="Dismiss"], #onetrust-accept-btn-handler, button:has-text("Akzeptieren"), button:has-text("Alle akzeptieren"), button[data-testid="selection-modal-close-btn"]' + ); + for (const btn of dismissBtns) { + await btn.click().catch(() => {}); + await page.waitForTimeout(300); } } catch (e) {} @@ -579,67 +568,35 @@ const worker = new Worker('scrape-jobs', async (job) => { if (isReviewScan) { console.log(`[Job ${job.id}] 🚀 Starte gezielte Review-Extraction für "${hotelDetails.name || 'Hotel'}"...`); - // A. Cookie-Banner und störende Overlays schließen - try { - const dismissSelectors = [ - '#onetrust-accept-btn-handler', - 'button#onetrust-reject-all-handler', - 'button[data-testid="selection-modal-close-btn"]', - 'button[aria-label="Schließen"]', - 'button[aria-label="Close"]' - ]; - for (const btnSel of dismissSelectors) { - const btn = await page.$(btnSel); - if (btn) { - await btn.click().catch(() => {}); - await page.waitForTimeout(400); - } - } - } catch (e) {} - // B. Herunterscrollen um Lazy-Components zu hydrieren und Review-Dialog zu öffnen console.log(`[Job ${job.id}] 📜 Scrolle herunter, um verzögert geladene Review-Komponenten zu aktivieren...`); await page.evaluate(() => window.scrollTo(0, 4000)); - await page.waitForTimeout(2000); + await page.waitForTimeout(2500); // C. Klick auf Review-Trigger ausführen - let isPanelOpen = false; - try { - console.log(`[Job ${job.id}] 👉 Öffne Review-Dialog über 'Alle Bewertungen lesen' / 'Gästebewertungen'...`); - isPanelOpen = await page.evaluate(() => { - const btn = document.querySelector('button[data-testid="fr-read-all-reviews"]') || - document.querySelector('button[data-testid="review-score-read-all-actionable"]') || - document.getElementById('reviews-tab-trigger') || - document.querySelector('[data-testid="Property-Header-Nav-Tab-Trigger-reviews"]'); - if (btn) { - btn.scrollIntoView({ behavior: 'instant', block: 'center' }); - btn.click(); - btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window })); - return true; - } - return false; - }).catch(() => false); - - // Warten bis das Dialog-Panel mit den 1.000+ Bewertungen gerendert ist - try { - await page.waitForSelector( - 'div[role="dialog"] [data-testid="review-card"], #reviewCardsSection [data-testid="review-card"], [data-testid="review-card"]', - { timeout: 15000 } - ); - console.log(`[Job ${job.id}] 🌟 Review-Dialog ('role="dialog"') mit allen Bewertungen erfolgreich geladen!`); - } catch (waitErr) { - console.log(`[Job ${job.id}] ⚠️ Dialog-Karten nicht sofort erkannt, fahre mit vorhandenem DOM fort...`); + console.log(`[Job ${job.id}] 👉 Öffne Review-Dialog über 'Alle Bewertungen lesen' / 'Gästebewertungen'...`); + await page.evaluate(() => { + const btn = document.querySelector('button[data-testid="fr-read-all-reviews"]') || + document.querySelector('button[data-testid="review-score-read-all-actionable"]') || + document.getElementById('reviews-tab-trigger') || + document.querySelector('[data-testid="Property-Header-Nav-Tab-Trigger-reviews"]'); + if (btn) { + btn.scrollIntoView({ behavior: 'instant', block: 'center' }); + btn.click(); + btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window })); + return true; } + return false; + }).catch(() => false); - // Screenshot zur visuellen Kontrolle aktualisieren - try { - await page.screenshot({ path: 'src/review-debug.png', fullPage: false }); - console.log(`[Job ${job.id}] 📸 Screenshot aktualisiert unter: scraper-service/src/review-debug.png`); - } catch (e) {} + // 4 Sekunden warten bis das Dialog-Panel mit den 1.000+ Bewertungen vollständig geöffnet ist + await page.waitForTimeout(4000); - } catch (e) { - console.log(`[Job ${job.id}] ⚠️ Fehler beim Klick auf Review-Trigger:`, e.message); - } + // Screenshot zur visuellen Kontrolle aktualisieren + try { + await page.screenshot({ path: 'src/review-debug.png', fullPage: false }); + console.log(`[Job ${job.id}] 📸 Screenshot aktualisiert unter: scraper-service/src/review-debug.png`); + } catch (e) {} // D. Sortierung auf "Neueste zuerst" (NEWEST_FIRST) umstellen try { @@ -660,84 +617,36 @@ const worker = new Worker('scrape-jobs', async (job) => { pageCount++; console.log(`[Job ${job.id}] 📄 Scanne Bewertungsseite ${pageCount} (Bisher gesammelt: ${extractedReviews.length})...`); - const evalResult = await safeEvaluate(page, ({ westernCountries }) => { - let cards = Array.from(document.querySelectorAll([ - '#reviewCardsSection [data-testid="review-card"]', - 'div[role="dialog"] [data-testid="review-card"]', - '[data-testid="review-card"]', - '[data-testid="review-cards"] > li', - '[data-testid="review-cards"] > div', - 'div[data-testid*="review-card"]', - 'li[data-testid*="review-card"]', - 'li.review_list_new_item_block', - 'div.c-review-block__item', - 'div[itemprop="review"]', - 'li.review_item', - 'div.c-review', - 'div.review_item', - '.c-review-block', - 'li.c-review-block' - ].join(','))); - - if (cards.length === 0) { - const container = document.querySelector('#hp-reviews-sliding, #reviewCardsSection, [data-testid="review-cards"], div[role="dialog"]'); - if (container) { - cards = Array.from(container.querySelectorAll('li, div.c-review-block, div.c-review')).filter(el => (el.innerText || '').length > 30); - } - } - - // Filterboxen und Scorecard-Zusammenfassungen explizit ausschließen - cards = cards.filter(el => { - const text = el.innerText || ''; - return text.length > 20 && - !text.includes('Wählen Sie Themen') && - !text.includes('Kategorien:') && - !el.classList.contains('review_score_breakdown'); - }); - - const results = []; - const debugLogs = []; + // Extrahiere alle Review-Karten aus dem Dialog / DOM + const pageReviews = await page.evaluate(() => { + const cards = Array.from(document.querySelectorAll( + '#reviewCardsSection [data-testid="review-card"], div[role="dialog"] [data-testid="review-card"], [data-testid="review-card"]' + )); + const items = []; const currentYear = 2026; - for (let idx = 0; idx < cards.length; idx++) { - const card = cards[idx]; - const cardText = card.innerText || ''; - if (!cardText.trim()) continue; - - // 1. Herkunftsland des Gastes ermitteln + for (const card of cards) { + const titleEl = card.querySelector('[data-testid="review-title"], h4'); + const posEl = card.querySelector('[data-testid="review-positive-text"]'); + const negEl = card.querySelector('[data-testid="review-negative-text"]'); + const dateEl = card.querySelector('[data-testid="review-date"]'); const flagImg = card.querySelector('[data-testid="review-avatar"] .fff1944c52 img[alt], [data-testid="review-country"] img[alt], img.b8d1620349'); const countrySpan = card.querySelector('[data-testid="review-avatar"] .fff1944c52 span:not(.cec4c9cfef), [data-testid="review-country"]'); - const guestCountryText = (flagImg ? flagImg.getAttribute('alt') : (countrySpan ? countrySpan.innerText : '')).trim(); - - if (guestCountryText && guestCountryText.length > 1) { - const isWestern = westernCountries.some(c => guestCountryText.toLowerCase().includes(c.toLowerCase())); - if (!isWestern) { - const preview = guestCountryText.substring(0, 50).replace(/\n/g, ' '); - debugLogs.push(`Karte #${idx + 1}: ❌ Land nicht in Liste ("${preview}")`); - continue; - } - } - - // 2. Datum / Jahr prüfen (letzte 2 Jahre) - const yearMatch = cardText.match(/\b(20\d{2})\b/); - if (yearMatch) { - const reviewYear = parseInt(yearMatch[1], 10); - if (currentYear - reviewYear > 2) { - debugLogs.push(`Karte #${idx + 1}: ❌ Zu alt (${reviewYear})`); - continue; - } - } - - // 3. Texte extrahieren - const titleEl = card.querySelector('[data-testid="review-title"], .c-review__title, h4, .review_title'); - const posEl = card.querySelector('[data-testid="review-positive-text"], .c-review__row--positive, .review_pos'); - const negEl = card.querySelector('[data-testid="review-negative-text"], .c-review__row--negative, .review_neg'); - const dateEl = card.querySelector('[data-testid="review-date"], .c-review__date, .review_item_date'); const title = titleEl ? titleEl.innerText.trim() : ''; const positive = posEl ? posEl.innerText.trim().replace(/^„|“$/g, '').trim() : ''; const negative = negEl ? negEl.innerText.trim().replace(/^„|“$/g, '').trim() : ''; - const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell'); + const dateText = dateEl ? dateEl.innerText.trim() : 'Aktuell'; + const country = (flagImg ? flagImg.getAttribute('alt') : (countrySpan ? countrySpan.innerText : '')).trim(); + + const cardText = card.innerText || ''; + const yearMatch = cardText.match(/\b(20\d{2})\b/); + if (yearMatch) { + const reviewYear = parseInt(yearMatch[1], 10); + if (currentYear - reviewYear > 2) { + continue; + } + } let fallbackText = ''; if (!positive && !negative) { @@ -745,47 +654,50 @@ const worker = new Worker('scrape-jobs', async (job) => { } if (title || positive || negative || fallbackText) { - results.push({ title, positive: positive || fallbackText, negative, date: dateText }); - debugLogs.push(`Karte #${idx + 1}: ✅ Akzeptiert ("${title || (positive ? positive.substring(0, 25) : 'Review')}")`); + items.push({ title, positive: positive || fallbackText, negative, date: dateText, country }); } } + return items; + }).catch(err => { + console.error(`[Job ${job.id}] Evaluate error on page ${pageCount}:`, err.message); + return []; + }); - return { - totalCards: cards.length, - results, - debugLogs: debugLogs.slice(0, 8) - }; - }, { totalCards: 0, results: [], debugLogs: [] }, { westernCountries: WESTERN_COUNTRIES }); + console.log(`[Job ${job.id}] 🔍 Echte Gästekarten auf Seite ${pageCount} gefunden: ${pageReviews.length}`); - console.log(`[Job ${job.id}] 🔍 Echte Gästekarten gefunden: ${evalResult.totalCards}`); - if (evalResult.debugLogs) { - evalResult.debugLogs.forEach(log => console.log(`[Job ${job.id}] └─ ${log}`)); - } - - // Deduplizierung - for (const rev of evalResult.results) { + for (const rev of pageReviews) { + if (rev.country) { + const isWestern = WESTERN_COUNTRIES.some(c => rev.country.toLowerCase().includes(c.toLowerCase())); + if (!isWestern) { + console.log(`[Job ${job.id}] └─ ❌ Land nicht in Liste: "${rev.country}"`); + continue; + } + } const key = `${rev.title}-${rev.positive}-${rev.date}`; if (!seenKeys.has(key) && extractedReviews.length < 80) { seenKeys.add(key); extractedReviews.push(rev); + console.log(`[Job ${job.id}] └─ ✅ Akzeptiert: "${rev.title || rev.positive.substring(0, 30)}" (${rev.country || 'International'})`); } } if (extractedReviews.length >= 80) break; - // Weiterblättern im Modal oder der Hauptseite - try { - const nextBtn = await page.$( - 'div[role="dialog"] button[aria-label="Nächste Seite"], button[aria-label="Nächste Seite"], button[aria-label*="Nächste Seite"], #hp-reviews-sliding button[aria-label*="Nächste"], button[aria-label*="Next"], #reviewCardsSection button:has(svg), a.paging-next, li.bui-pagination__next-arrow a' - ); - if (nextBtn) { - const isDisabled = await nextBtn.getAttribute('disabled'); - if (isDisabled === null) { - await nextBtn.click({ force: true }); - await page.waitForTimeout(2500); - } else break; - } else break; - } catch (e) { + // Weiterblättern im Modal + const nextResult = await page.evaluate(() => { + const nextBtn = document.querySelector('div[role="dialog"] button[aria-label="Nächste Seite"], button[aria-label="Nächste Seite"], button[aria-label*="Nächste Seite"]'); + if (nextBtn && !nextBtn.disabled && !nextBtn.hasAttribute('disabled')) { + nextBtn.scrollIntoView({ behavior: 'instant', block: 'center' }); + nextBtn.click(); + nextBtn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window })); + return true; + } + return false; + }).catch(() => false); + + if (nextResult) { + await page.waitForTimeout(3000); + } else { break; } }