From a5819a0d73fda65571afb72199035e2b93438a05 Mon Sep 17 00:00:00 2001 From: gitea Date: Sun, 9 Aug 2026 21:55:01 +0000 Subject: [PATCH] Review funkt --- src/index.js | 291 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 205 insertions(+), 86 deletions(-) diff --git a/src/index.js b/src/index.js index 9fd50c8..af516c5 100644 --- a/src/index.js +++ b/src/index.js @@ -1,38 +1,73 @@ +import { Worker } from 'bullmq'; +import { chromium } from 'playwright'; +import axios from 'axios'; + +const REDIS_CONFIG = { + connection: { + host: process.env.REDIS_HOST || 'redis', + port: parseInt(process.env.REDIS_PORT || '6379', 10) + } +}; + +const WESTERN_COUNTRIES = [ + 'Deutschland', 'Schweiz', 'Österreich', 'Vereinigte Staaten', 'USA', + 'Vereinigtes Königreich', 'Großbritannien', 'Niederlande', 'Frankreich', + 'Spanien', 'Italien', 'Kanada', 'Australien', 'Belgien', 'Schweden', + 'Norwegen', 'Dänemark', 'Finnland', 'Irland', 'Luxemburg', 'Indien' +]; + const worker = new Worker('scrape-jobs', async (job) => { - const { url, webhookUrl } = job.data; + const { url, webhookUrl, appWebhookUrl, hotelId } = job.data; console.log(`[Job ${job.id}] Starte Scraping für: ${url}`); + // Variablen auf Worker-Ebene deklarieren (Verhindert ReferenceError in finally) let scrapedPrice = null; let roomSizeM2 = null; let hotelDetails = { name: null, rating: null, reviewCount: null, address: null, image: null }; + let domData = { dNights: 1, dPersons: 2, roomSize: null }; + let extractedReviews = []; let browser = null; - // 1. Fallback: Nächte aus URL-Checkin/Checkout berechnen - const urlObj = new URL(url); - const checkin = urlObj.searchParams.get('checkin'); - const checkout = urlObj.searchParams.get('checkout'); - const adultsParam = parseInt(urlObj.searchParams.get('group_adults') || '0', 10); - - let nightsFromUrl = null; - if (checkin && checkout) { - const diffTime = Math.abs(new Date(checkout) - new Date(checkin)); - nightsFromUrl = Math.ceil(diffTime / (1000 * 60 * 60 * 24)); - } - try { browser = await chromium.launch({ headless: true }); const context = await browser.newContext({ userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', - viewport: { width: 1280, height: 800 }, + viewport: { width: 1280, height: 900 }, locale: 'de-DE', timezoneId: 'Europe/Berlin' }); const page = await context.newPage(); - await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 35000 }); - await page.waitForTimeout(3000); - // 2. Hotel Details aus JSON-LD + let targetUrl = url; + if (!targetUrl.includes('#tab-reviews')) { + targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews'; + } + if (!targetUrl.includes('sort_by=')) { + targetUrl += (targetUrl.includes('?') ? '&' : '?') + 'sort_by=f_recent_desc'; + } + + await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 35000 }); + await page.waitForTimeout(2500); + + // Klick auf Bewertungs-Reiter erzwingen + try { + const reviewTab = await page.$('#js--hp-gallery-scorecard a, a[href*="#tab-reviews"], [data-testid="review-score-link"]'); + if (reviewTab) { + await reviewTab.click(); + await page.waitForTimeout(2000); + } + } catch (e) {} + + // Scrollen + await page.evaluate(() => { + const el = document.querySelector('#reviewListSorters') || document.querySelector('#tab-reviews'); + if (el) el.scrollIntoView(); + else window.scrollBy(0, 1000); + }); + await page.waitForTimeout(2500); + + // 1. JSON-LD Stammdaten hotelDetails = await page.evaluate(() => { const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]')); for (const script of scripts) { @@ -52,94 +87,178 @@ const worker = new Worker('scrape-jobs', async (job) => { return { name: null, rating: null, reviewCount: null, address: null, image: null }; }); - // 3. Nächte, Personen & Zimmergröße direkt aus den Booking-DOM-Selektoren lesen - const domData = await page.evaluate(() => { + // 2. Preis & DOM-Daten + domData = await page.evaluate(() => { let dNights = null; let dPersons = null; - - // Gezielte Suche in den von dir genannten Selektoren - const targetSelectors = [ - '.bui-price-display__label', - '#searchbox-horizontal-occupancy-label', - '[data-testid="price-for-x-nights"]', - '#group_recommendation .bui-price-display__label', - '[data-testid="occupancy-config"]' - ]; - + const targetSelectors = ['.bui-price-display__label', '#searchbox-horizontal-occupancy-label', '[data-testid="price-for-x-nights"]']; let combinedText = ''; - for (const sel of targetSelectors) { - const els = document.querySelectorAll(sel); - els.forEach(el => { - if (el && el.innerText) combinedText += ' ' + el.innerText; - }); - } + targetSelectors.forEach(s => document.querySelectorAll(s).forEach(el => combinedText += ' ' + el.innerText)); + if (!combinedText.trim()) combinedText = document.body.innerText; - // Falls Selektoren leer sind, gesamten Text als Fallback nutzen - if (!combinedText.trim()) { - combinedText = document.body.innerText; - } + const nMatch = combinedText.match(/(\d+)\s*Nächte?/i); + if (nMatch) dNights = parseInt(nMatch[1], 10); + const pMatch = combinedText.match(/(\d+)\s*(?:Erwachsene|Erwachsener|Gäste|Personen)/i); + if (pMatch) dPersons = parseInt(pMatch[1], 10); + const rmMatch = document.body.innerText.match(/(\d+)\s*m²/i); - // Nächte matchen (z. B. "7 Nächte", "1 Nacht", "für 2 Nächte") - const nightMatch = combinedText.match(/(\d+)\s*Nächte?/i); - if (nightMatch) dNights = parseInt(nightMatch[1], 10); - - // Personen matchen (z. B. "2 Erwachsene", "1 Erwachsener", "2 Gäste") - const personMatch = combinedText.match(/(\d+)\s*(?:Erwachsene|Erwachsener|Gäste|Personen)/i); - if (personMatch) dPersons = parseInt(personMatch[1], 10); - - // Zimmergröße (m²) - const roomMatch = document.body.innerText.match(/(\d+)\s*m²/i); - const roomSize = roomMatch ? parseInt(roomMatch[1], 10) : null; - - return { dNights, dPersons, roomSize }; + return { dNights, dPersons, roomSize: rmMatch ? parseInt(rmMatch[1], 10) : null }; }); roomSizeM2 = domData.roomSize; - // Finale Werte bestimmen (Priorität: URL -> DOM -> Standard-Fallback) - const finalNights = nightsFromUrl || domData.dNights || 1; - const finalPersons = adultsParam || domData.dPersons || 2; - - // 4. Preis extrahieren - const priceSelectors = [ - '[data-testid="price-and-discounted-price"]', - '.totalPrice .bui-price-display__value', - '.hprt-price-price' - ]; - - for (const selector of priceSelectors) { - const el = await page.$(selector); + const priceSelectors = ['[data-testid="price-and-discounted-price"]', '.totalPrice .bui-price-display__value', '.hprt-price-price']; + for (const sel of priceSelectors) { + const el = await page.$(sel); if (el) { const text = await el.innerText(); - if (text && /\d/.test(text)) { - scrapedPrice = text.trim(); - break; - } + if (text && /\d/.test(text)) { scrapedPrice = text.trim(); break; } } } - // 5. Webhook mit exakten Daten absenden - if (webhookUrl && webhookUrl.startsWith('http')) { - await axios.post(webhookUrl, { - jobId: job.id, - url, - hotelName: hotelDetails.name, - price: scrapedPrice, - roomSizeM2, - rating: hotelDetails.rating, - reviewCount: hotelDetails.reviewCount, - address: hotelDetails.address, - image: hotelDetails.image, - nights: finalNights, - persons: finalPersons, - timestamp: new Date().toISOString() - }); + // 3. KOMMENTARE MIT PAGINATION BIS ZU 80 STÜCK EXTRAHIEREN + let pageCount = 0; + const seenKeys = new Set(); + + while (extractedReviews.length < 80 && pageCount < 15) { + const pageReviews = await page.evaluate(({ westernCountries }) => { + const dateElements = Array.from(document.querySelectorAll('[data-testid="review-date"]')); + const results = []; + const currentYear = 2026; + + for (const dateEl of dateElements) { + // Navigiere hoch zum Container, der SOWOHL Person/Land ALS AUCH Bewertungstext enthält + let card = dateEl.parentElement; + while (card && card !== document.body) { + if (card.querySelector('[data-testid="review-avatar"]') || card.querySelector('[aria-label="Bewertende Person"]')) { + break; + } + card = card.parentElement; + } + if (!card || card === document.body) { + card = dateEl.closest('li') || dateEl.parentElement.parentElement.parentElement; + } + + const cardText = card ? card.innerText : ''; + if (!cardText) continue; + + // Herkunftsland prüfen + const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase())); + if (!isWestern) continue; + + // Datum prüfen (<= 2 Jahre) + const dateText = dateEl.innerText || ''; + const yearMatch = cardText.match(/\b(20\d{2})\b/); + if (yearMatch) { + const reviewYear = parseInt(yearMatch[1], 10); + if (currentYear - reviewYear > 2) continue; + } + + const titleEl = card.querySelector('[data-testid="review-title"]'); + const posEl = card.querySelector('[data-testid="review-positive-text"], .review_pos'); + const negEl = card.querySelector('[data-testid="review-negative-text"], .review_neg'); + + const title = titleEl ? titleEl.innerText.trim() : ''; + const positive = posEl ? posEl.innerText.trim() : ''; + const negative = negEl ? negEl.innerText.trim() : ''; + + let generalComment = ''; + if (!positive && !negative) { + const commentEl = card.querySelector('.b99b6ef58f, .a17eaf5728'); + if (commentEl && !commentEl.innerText.includes('keinen Kommentar')) { + generalComment = commentEl.innerText.trim(); + } + } + + if (title || positive || negative || generalComment) { + results.push({ + title, + positive: positive || generalComment, + negative, + date: dateText + }); + } + } + + return results; + }, { westernCountries: WESTERN_COUNTRIES }); + + // Duplikate vermeiden + for (const rev of pageReviews) { + const key = `${rev.title}-${rev.positive}-${rev.date}`; + if (!seenKeys.has(key) && extractedReviews.length < 80) { + seenKeys.add(key); + extractedReviews.push(rev); + } + } + + if (extractedReviews.length >= 80) break; + + // Klick auf "Nächste Seite" Pfeil + try { + const nextBtn = await page.$( + '#reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"], [data-testid="pagination-next-page"]' + ); + + if (nextBtn) { + const isDisabled = await nextBtn.getAttribute('disabled'); + if (isDisabled !== null) break; + + await nextBtn.click(); + await page.waitForTimeout(2500); + pageCount++; + } else { + break; + } + } catch (e) { + break; + } } + console.log(`[Job ${job.id}] ${extractedReviews.length} Kommentare über ${pageCount + 1} Seiten extrahiert.`); + } catch (error) { - console.error(`[Job ${job.id}] Fehler:`, error.message); + console.error(`[Job ${job.id}] Fehler beim Scraping:`, error.message); } finally { if (browser) await browser.close(); + + // 1. PREISHISTORIE IN POSTGRES SPEICHERN + if (appWebhookUrl) { + try { + await axios.post(appWebhookUrl, { + price: scrapedPrice, + roomSizeM2, + hotelName: hotelDetails.name, + rating: hotelDetails.rating, + reviewCount: hotelDetails.reviewCount, + address: hotelDetails.address, + image: hotelDetails.image, + nights: domData.dNights || 1, + persons: domData.dPersons || 2, + }); + console.log(`[Job ${job.id}] ✅ Preishistorie direkt in Postgres gespeichert!`); + } catch (e) { + console.error(`[Job ${job.id}] ❌ Fehler beim Speichern der Preishistorie:`, e.message); + } + } + + // 2. REVIEWS AN N8N SENDEN + if (webhookUrl) { + try { + await axios.post(webhookUrl, { + jobId: job.id, + hotelId, + hotelName: hotelDetails.name, + reviews: extractedReviews, + timestamp: new Date().toISOString() + }); + console.log(`[Job ${job.id}] 🚀 ${extractedReviews.length} Reviews an n8n gesendet!`); + } catch (e) { + console.error(`[Job ${job.id}] ❌ Fehler beim Senden an n8n:`, e.message); + } + } } }, { ...REDIS_CONFIG, concurrency: 2 }); + +console.log('🚀 Scraper Worker bereit...');