diff --git a/src/index.js b/src/index.js index 8d517be..8a046e0 100644 --- a/src/index.js +++ b/src/index.js @@ -2,6 +2,9 @@ import { Worker } from 'bullmq'; import { chromium } from 'playwright'; import axios from 'axios'; +// ========================================== +// KONFIGURATION & LISTEN +// ========================================== const REDIS_CONFIG = { connection: { host: process.env.REDIS_HOST || 'redis', @@ -12,37 +15,34 @@ const REDIS_CONFIG = { const SERPAPI_KEY = process.env.SERPAPI_KEY || null; const WESTERN_COUNTRIES = [ - 'Deutschland', 'Schweiz', 'Österreich', 'Vereinigte Staaten', 'USA', - 'Vereinigtes Königreich', 'Großbritannien', 'Niederlande', 'Frankreich', - 'Spanien', 'Italien', 'Kanada', 'Australien', 'Belgien', 'Schweden', - 'Norwegen', 'Dänemark', 'Finnland', 'Irland', 'Luxemburg', 'Indien' + 'Deutschland', 'Schweiz', 'Österreich', 'Germany', 'Switzerland', 'Austria', + 'Vereinigte Staaten', 'USA', 'United States', 'Vereinigtes Königreich', 'Großbritannien', + 'United Kingdom', 'UK', 'Niederlande', 'Netherlands', 'Frankreich', 'France', + 'Spanien', 'Spain', 'Italien', 'Italy', 'Kanada', 'Canada', 'Australien', 'Australia', + 'Belgien', 'Belgium', 'Schweden', 'Sweden', 'Norwegen', 'Norway', 'Dänemark', 'Denmark', + 'Finnland', 'Finland', 'Irland', 'Ireland', 'Luxemburg', 'Luxembourg', 'Indien', 'India' ]; -// 1. Google Maps Fotos via SerpApi abrufen (inkl. Adressbereinigung & Fallback) +// ========================================== +// HILFSFUNKTION: GOOGLE MAPS BAD-FOTOS VIA SERPAPI +// ========================================== async function getGoogleBathroomPhotosViaApi(hotelName, hotelAddress) { if (!SERPAPI_KEY) { - console.log('⚠️ [SerpApi] Kein SERPAPI_KEY in der .env hinterlegt!'); + console.log('ℹ️ [SerpApi] Kein SERPAPI_KEY konfiguriert – überspringe Google-Maps-Fotos.'); return []; } try { - // Booking-Adresse bereinigen: Nur Stadt und Land nutzen für maximale Trefferquote let cleanLocation = ''; if (hotelAddress) { const parts = hotelAddress.split(',').map(p => p.trim()); cleanLocation = parts.length > 1 ? parts.slice(-2).join(' ') : hotelAddress; } const query = `${hotelName} ${cleanLocation}`.trim(); - console.log(`🔍 [SerpApi] Suche Google Maps Place für: "${query}"...`); + console.log(`🔍 [SerpApi] Suche Google Maps Ort für: "${query}"...`); - // A: Google Maps Ort suchen const searchRes = await axios.get('https://serpapi.com/search.json', { - params: { - engine: 'google_maps', - q: query, - hl: 'de', - api_key: SERPAPI_KEY - }, + params: { engine: 'google_maps', q: query, hl: 'de', api_key: SERPAPI_KEY }, timeout: 15000 }); @@ -55,73 +55,53 @@ async function getGoogleBathroomPhotosViaApi(hotelName, hotelAddress) { return []; } - console.log(`📸 [SerpApi] data_id gefunden (${dataId}). Lade Fotoliste...`); - - // B: Fotos & Kategorien abrufen const photosRes = await axios.get('https://serpapi.com/search.json', { - params: { - engine: 'google_maps_photos', - data_id: dataId, - hl: 'de', - api_key: SERPAPI_KEY - }, + params: { engine: 'google_maps_photos', data_id: dataId, hl: 'de', api_key: SERPAPI_KEY }, timeout: 15000 }); const categories = photosRes.data.categories || []; - // Flexible Erkennung für Bad-, Zimmer- und Innen-Tags + const strictBathKeywords = ['badezimmer', 'bad', 'bathroom', 'wc', 'toilet', 'washroom', 'restroom']; const bathCat = categories.find(c => { - const title = (c.title || '').toLowerCase(); - return title.includes('bad') || title.includes('zimmer') || title.includes('room') || title.includes('innen') || title.includes('gast'); + const title = (c.title || '').toLowerCase().trim(); + return strictBathKeywords.some(kw => title.includes(kw)); }); - let photosList = photosRes.data.photos || []; - - // Falls Kategorie vorhanden, gezielt Unterkategorie laden + let photosList = []; if (bathCat?.id) { - console.log(`🚽 [SerpApi] Kategorie gefunden: "${bathCat.title}"`); - try { - const catRes = await axios.get('https://serpapi.com/search.json', { - params: { - engine: 'google_maps_photos', - data_id: dataId, - category_id: bathCat.id, - hl: 'de', - api_key: SERPAPI_KEY - }, - timeout: 15000 - }); - if (catRes.data.photos && catRes.data.photos.length > 0) { - photosList = catRes.data.photos; - } - } catch (e) { - console.log('⚠️ [SerpApi] Fallback auf allgemeine Fotos aktiv.'); - } + console.log(`🚽 [SerpApi] Bad-Kategorie gefunden: "${bathCat.title}"`); + const catRes = await axios.get('https://serpapi.com/search.json', { + params: { engine: 'google_maps_photos', data_id: dataId, category_id: bathCat.id, hl: 'de', api_key: SERPAPI_KEY }, + timeout: 15000 + }); + if (catRes.data.photos?.length > 0) photosList = catRes.data.photos; + } else { + console.log('ℹ️ [SerpApi] Kein separater Bad-Ordner auf Google Maps vorhanden.'); + return []; } - // URLs bereinigen und auf HD-Auflösung setzen - const imageUrls = photosList + return photosList .map(p => p.image || p.thumbnail) .filter(Boolean) .map(url => url.replace(/=w\d+-h\d+[^=]*/, '=w1200-h800-k-no')) .slice(0, 12); - - console.log(`✅ [SerpApi] ${imageUrls.length} Gästefotos erfolgreich geladen.`); - return imageUrls; - } catch (error) { - console.error('❌ [SerpApi] Fehler:', error.response?.data?.error || error.message); + console.error('❌ [SerpApi] Fehler beim Fotoabruf:', error.message); return []; } } -// 2. Worker Definition +// ========================================== +// BULLMQ WORKER: SCRAPING & REVIEWS +// ========================================== const worker = new Worker('scrape-jobs', async (job) => { const { url, webhookUrl, appWebhookUrl, hotelId } = job.data; const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http')); - console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews + Bad-Fotos)' : 'PREIS-SCAN (Schnell)'} für: ${url}`); - // Alle Variablen fest vorinitialisieren + console.log(`\n==================================================`); + console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews + Fotos)' : 'PREIS-SCAN (Schnell)'} für: ${url}`); + console.log(`==================================================`); + let scrapedPrice = null; let hasGenius = false; let roomSizeM2 = null; @@ -132,21 +112,37 @@ const worker = new Worker('scrape-jobs', async (job) => { let browser = null; try { - browser = await chromium.launch({ headless: true }); + // 1. Browser mit Anti-Bot & Stealth-Optionen starten + browser = await chromium.launch({ + headless: true, + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-blink-features=AutomationControlled', + '--disable-infobars' + ] + }); + const context = await browser.newContext({ - userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', - viewport: { width: 1280, height: 900 }, + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', + viewport: { width: 1366, height: 900 }, locale: 'de-DE', timezoneId: 'Europe/Berlin' }); + const page = await context.newPage(); + // Verhindert Erkennung über navigator.webdriver + await page.addInitScript(() => { + Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); + }); + let targetUrl = url; if (isReviewScan && !targetUrl.includes('#tab-reviews')) { targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews'; } - await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 35000 }); + await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 40000 }); await page.waitForTimeout(2500); // DATUM AUS URL PARSEN @@ -156,7 +152,7 @@ const worker = new Worker('scrape-jobs', async (job) => { domData.checkoutDate = parsedUrl.searchParams.get('checkout'); } catch (e) {} - // 1. JSON-LD STAMMDATEN + // 2. JSON-LD STAMMDATEN EXTRAHIEREN hotelDetails = await page.evaluate(() => { try { const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]')); @@ -178,7 +174,7 @@ const worker = new Worker('scrape-jobs', async (job) => { return { name: null, rating: null, reviewCount: null, address: null, image: null }; }); - // 2. DOM DATEN + // 3. DOM-DATEN (Nächte, Personen, Zimmergröße) const extraDom = await page.evaluate(() => { try { const bodyText = document.body ? (document.body.innerText || '') : ''; @@ -214,7 +210,7 @@ const worker = new Worker('scrape-jobs', async (job) => { return null; }); - // GENIUS BADGE ERKENNUNG + // GENIUS BADGE PRÜFUNG hasGenius = await page.evaluate(() => { const geniusSelectors = [ '.m-badge__genius', @@ -231,44 +227,175 @@ const worker = new Worker('scrape-jobs', async (job) => { console.log(`[Job ${job.id}] 💙 Genius-Badge erkannt (-10% Rabatt gilt)!`); } - // 3. KOMMENTARE EXTRAHIEREN +// 4. REVIEWS EXTRAHIEREN (STICKY-HEADER AKTIVIERUNG & ECHTER MAUSKLICK) if (isReviewScan) { - console.log(`[Job ${job.id}] Starte Review-Extraction mit Pagination...`); + console.log(`[Job ${job.id}] 🚀 Starte gezielte Review-Extraction für "${hotelDetails.name || 'Hotel'}"...`); + + // A. Cookie-Banner und störende Overlays schließen try { - const reviewTab = await page.$('a[href*="#tab-reviews"], #js--hp-gallery-scorecard a, [data-testid="review-score-link"]'); - if (reviewTab) { - await reviewTab.click(); - await page.waitForTimeout(2000); + const dismissSelectors = [ + '#onetrust-accept-btn-handler', + 'button#onetrust-reject-all-handler', + 'button[data-testid="selection-modal-close-btn"]', + 'button[aria-label="Schließen"]', + 'button[aria-label="Close"]' + ]; + for (const btnSel of dismissSelectors) { + const btn = await page.$(btnSel); + if (btn) { + await btn.click().catch(() => {}); + await page.waitForTimeout(400); + } } } catch (e) {} - await page.evaluate(() => window.scrollBy(0, 1000)); - await page.waitForTimeout(2000); + // B. 500px nach unten scrollen, damit Bookings Sticky-Header (#reviews-tab-trigger) überhaupt sichtbar & aktiv wird + await page.evaluate(() => window.scrollBy(0, 550)); + await page.waitForTimeout(1500); + + // C. Klick per echtem Maus-Event auf den Trigger ausführen + let clicked = false; + try { + // Liste möglicher Klick-Ziele (Header-Tab, Scorecard-Badge oder Text "Gästebewertungen") + const triggerLocators = [ + page.locator('#reviews-tab-trigger').first(), + page.locator('[data-testid="Property-Header-Nav-Tab-Trigger-reviews"]').first(), + page.locator('a:has-text("Gästebewertungen")').first(), + page.locator('button:has-text("Gästebewertungen")').first(), + page.locator('[data-testid="review-score-right-component"]').first(), + page.locator('[data-testid="review-score-component"]').first() + ]; + + for (const loc of triggerLocators) { + if (await loc.count() > 0 && await loc.isVisible()) { + console.log(`[Job ${job.id}] 👉 Sichtbaren Review-Trigger gefunden. Führe Klick aus...`); + + // Reale Maus-Koordinaten ermitteln & klicken + const box = await loc.boundingBox(); + if (box) { + await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2); + clicked = true; + await page.waitForTimeout(3500); + break; + } else { + await loc.click({ force: true }).catch(() => {}); + clicked = true; + await page.waitForTimeout(3500); + break; + } + } + } + + // Fallback: JS-Klick direkt auf das HTML-Element mit Text "Gästebewertungen" + if (!clicked) { + console.log(`[Job ${job.id}] ⚠️ Fallback: Klick per DOM-Suche nach 'Gästebewertungen'...`); + clicked = await page.evaluate(() => { + const el = Array.from(document.querySelectorAll('a, button, span, div')) + .find(e => e.innerText && e.innerText.includes('Gästebewertungen') && (e.id === 'reviews-tab-trigger' || e.closest('#reviews-tab-trigger') || e.getAttribute('data-testid')?.includes('reviews'))); + if (el) { + const clickTarget = el.closest('a') || el.closest('button') || el; + clickTarget.click(); + clickTarget.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window })); + return true; + } + return false; + }).catch(() => false); + + if (clicked) await page.waitForTimeout(3500); + } + + // Screenshot zur visuellen Kontrolle aktualisieren + try { + await page.screenshot({ path: 'src/review-debug.png', fullPage: false }); + console.log(`[Job ${job.id}] 📸 Screenshot aktualisiert unter: scraper-service/src/review-debug.png`); + } catch (e) {} + + } catch (e) { + console.log(`[Job ${job.id}] ⚠️ Fehler beim Klick auf Review-Trigger:`, e.message); + } + + // Warten bis Review-Karten im Sliding Panel oder DOM sichtbar sind + try { + await page.waitForSelector( + '[data-testid="review-card"], [data-testid="review-cards"], #hp-reviews-sliding, div[data-component="hotel/reviews/review-card"], li.review_item', + { timeout: 8000 } + ); + console.log(`[Job ${job.id}] 👁️ Review-Bereich erfolgreich gerendert.`); + } catch (e) { + console.log(`[Job ${job.id}] ⚠️ Wartezeit abgelaufen – starte Auslesen aus dem aktuellen DOM...`); + } let pageCount = 0; const seenKeys = new Set(); - while (extractedReviews.length < 80 && pageCount < 15) { - const pageReviews = await page.evaluate(({ westernCountries }) => { - const cards = Array.from(document.querySelectorAll('[data-testid="review-card"], [role="group"][aria-label*="Bewertung"], li.review_list_new_item_block, .review_item')); - const results = []; - const currentYear = 2026; - for (const card of cards) { - const cardText = card ? (card.innerText || '') : ''; - if (!cardText) continue; - const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase())); - if (!isWestern) continue; + while (pageCount < 30) { + pageCount++; + console.log(`[Job ${job.id}] 📄 Scanne Bewertungsseite ${pageCount} (Bisher gesammelt: ${extractedReviews.length})...`); + const evalResult = await page.evaluate(({ westernCountries }) => { + let cards = Array.from(document.querySelectorAll([ + '[data-testid="review-card"]', + '[data-testid="review-cards"] > li', + '[data-testid="review-cards"] > div', + 'div[data-testid*="review-card"]', + 'li[data-testid*="review-card"]', + 'li.review_list_new_item_block', + 'div.c-review-block__item', + 'div[itemprop="review"]', + 'li.review_item' + ].join(','))); + + if (cards.length === 0) { + const container = document.querySelector('#hp-reviews-sliding, #reviewCardsSection, [data-testid="review-cards"], #blockdisplay4'); + if (container) { + cards = Array.from(container.querySelectorAll('li, div')).filter(el => (el.innerText || '').length > 40); + } + } + + // Filterboxen und Scorecard-Zusammenfassungen explizit ausschließen + cards = cards.filter(el => { + const text = el.innerText || ''; + return text.length > 25 && + !text.includes('Wählen Sie Themen') && + !text.includes('Kategorien:') && + !el.classList.contains('review_score_breakdown'); + }); + + const results = []; + const debugLogs = []; + const currentYear = 2026; + + for (let idx = 0; idx < cards.length; idx++) { + const card = cards[idx]; + const cardText = card.innerText || ''; + if (!cardText.trim()) continue; + + // 1. Herkunftsland des Gastes ermitteln + const countryEl = card.querySelector('[data-testid="review-country"], .bui-avatar-or-image__title, .user_country, .c-review__country'); + const guestCountryText = countryEl ? countryEl.innerText : cardText; + + const isWestern = westernCountries.some(c => guestCountryText.toLowerCase().includes(c.toLowerCase())); + if (!isWestern) { + const preview = guestCountryText.substring(0, 50).replace(/\n/g, ' '); + debugLogs.push(`Karte #${idx + 1}: ❌ Land nicht in Liste ("${preview}")`); + continue; + } + + // 2. Datum / Jahr prüfen (letzte 2 Jahre) const yearMatch = cardText.match(/\b(20\d{2})\b/); if (yearMatch) { const reviewYear = parseInt(yearMatch[1], 10); - if (currentYear - reviewYear > 2) continue; + if (currentYear - reviewYear > 2) { + debugLogs.push(`Karte #${idx + 1}: ❌ Zu alt (${reviewYear})`); + continue; + } } - const titleEl = card.querySelector('[data-testid="review-title"], h4'); - const posEl = card.querySelector('[data-testid="review-positive-text"], .review_pos'); - const negEl = card.querySelector('[data-testid="review-negative-text"], .review_neg'); - const dateEl = card.querySelector('[data-testid="review-date"]'); + // 3. Texte extrahieren + const titleEl = card.querySelector('[data-testid="review-title"], .c-review__title, h4, .review_title'); + const posEl = card.querySelector('[data-testid="review-positive-text"], .c-review__row--positive, .review_pos'); + const negEl = card.querySelector('[data-testid="review-negative-text"], .c-review__row--negative, .review_neg'); + const dateEl = card.querySelector('[data-testid="review-date"], .c-review__date, .review_item_date'); const title = titleEl ? titleEl.innerText.trim() : ''; const positive = posEl ? posEl.innerText.trim() : ''; @@ -276,16 +403,30 @@ const worker = new Worker('scrape-jobs', async (job) => { const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell'); let fallbackText = ''; - if (!positive && !negative) fallbackText = cardText.replace(title, '').substring(0, 300).trim(); + if (!positive && !negative) { + fallbackText = cardText.replace(title, '').substring(0, 300).trim(); + } if (title || positive || negative || fallbackText) { results.push({ title, positive: positive || fallbackText, negative, date: dateText }); + debugLogs.push(`Karte #${idx + 1}: ✅ Akzeptiert ("${title || (positive ? positive.substring(0, 25) : 'Review')}")`); } } - return results; - }, { westernCountries: WESTERN_COUNTRIES }).catch(() => []); - for (const rev of pageReviews) { + return { + totalCards: cards.length, + results, + debugLogs: debugLogs.slice(0, 8) + }; + }, { westernCountries: WESTERN_COUNTRIES }); + + console.log(`[Job ${job.id}] 🔍 Echte Gästekarten gefunden: ${evalResult.totalCards}`); + if (evalResult.debugLogs) { + evalResult.debugLogs.forEach(log => console.log(`[Job ${job.id}] └─ ${log}`)); + } + + // Deduplizierung + for (const rev of evalResult.results) { const key = `${rev.title}-${rev.positive}-${rev.date}`; if (!seenKeys.has(key) && extractedReviews.length < 80) { seenKeys.add(key); @@ -295,26 +436,32 @@ const worker = new Worker('scrape-jobs', async (job) => { if (extractedReviews.length >= 80) break; + // Weiterblättern im Modal oder der Hauptseite try { - const nextBtn = await page.$('#reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"]'); + const nextBtn = await page.$( + '#hp-reviews-sliding button[aria-label*="Nächste"], #hp-reviews-sliding button[aria-label*="Next"], #reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"], a.paging-next, li.bui-pagination__next-arrow a' + ); if (nextBtn) { const isDisabled = await nextBtn.getAttribute('disabled'); - if (isDisabled !== null) break; - await nextBtn.click(); - await page.waitForTimeout(2000); - pageCount++; + if (isDisabled === null) { + await nextBtn.click({ force: true }); + await page.waitForTimeout(2500); + } else break; } else break; - } catch (e) { break; } + } catch (e) { + break; + } } - console.log(`[Job ${job.id}] ${extractedReviews.length} Kommentare extrahiert.`); + + console.log(`[Job ${job.id}] ✅ Insgesamt ${extractedReviews.length} passende Kommentare extrahiert.`); } } catch (error) { - console.error(`[Job ${job.id}] Fehler beim Scraping:`, error.message); + console.error(`[Job ${job.id}] ❌ Fehler beim Scraping:`, error.message); } finally { if (browser) await browser.close(); -// 4. Google Maps Fotos nur bei gezieltem KI-Review-Scan abrufen (spart SerpApi-Credits) + // 5. GOOGLE MAPS FOTOS NUR BEI KI-REVIEW SCAN ABRUFEN if (isReviewScan && hotelDetails.name) { try { bathroomImages = await getGoogleBathroomPhotosViaApi(hotelDetails.name, hotelDetails.address); @@ -324,7 +471,7 @@ const worker = new Worker('scrape-jobs', async (job) => { } } - // 5. Preishistorie und Bad-Bilder an Travel-App Webhook senden + // 6. PREISHISTORIE + STAMMDATEN AN NEXT.JS WEBHOOK SENDEN if (appWebhookUrl) { try { await axios.post(appWebhookUrl, { @@ -342,13 +489,13 @@ const worker = new Worker('scrape-jobs', async (job) => { persons: domData.dPersons || 2, bathroomImages: Array.isArray(bathroomImages) && bathroomImages.length > 0 ? bathroomImages : undefined }); - console.log(`[Job ${job.id}] ✅ Preishistorie in Postgres gespeichert!`); + console.log(`[Job ${job.id}] ✅ Preishistorie & Stammdaten in Postgres gespeichert!`); } catch (e) { - console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie:`, e.message); + console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie-Webhook:`, e.message); } } - // 6. Reviews an n8n senden + // 7. REVIEWS AN N8N WEBHOOK SENDEN if (isReviewScan && extractedReviews.length > 0) { try { await axios.post(webhookUrl, { @@ -359,7 +506,9 @@ const worker = new Worker('scrape-jobs', async (job) => { timestamp: new Date().toISOString() }); console.log(`[Job ${job.id}] 🚀 ${extractedReviews.length} Reviews an n8n gesendet!`); - } catch (e) {} + } catch (e) { + console.error(`[Job ${job.id}] ❌ Fehler beim Senden an n8n:`, e.message); + } } } }, { ...REDIS_CONFIG, concurrency: 2 }); diff --git a/src/review-debug.png b/src/review-debug.png new file mode 100644 index 0000000..367ecac Binary files /dev/null and b/src/review-debug.png differ