Review funkt

src/index.js

@@ -1,38 +1,73 @@
+import { Worker } from 'bullmq';
+import { chromium } from 'playwright';
+import axios from 'axios';
+
+const REDIS_CONFIG = {
+  connection: {
+    host: process.env.REDIS_HOST || 'redis',
+    port: parseInt(process.env.REDIS_PORT || '6379', 10)
+  }
+};
+
+const WESTERN_COUNTRIES = [
+  'Deutschland', 'Schweiz', 'Österreich', 'Vereinigte Staaten', 'USA', 
+  'Vereinigtes Königreich', 'Großbritannien', 'Niederlande', 'Frankreich', 
+  'Spanien', 'Italien', 'Kanada', 'Australien', 'Belgien', 'Schweden', 
+  'Norwegen', 'Dänemark', 'Finnland', 'Irland', 'Luxemburg', 'Indien'
+];
+
 const worker = new Worker('scrape-jobs', async (job) => {
-  const { url, webhookUrl } = job.data;
+  const { url, webhookUrl, appWebhookUrl, hotelId } = job.data;
   console.log(`[Job ${job.id}] Starte Scraping für: ${url}`);
 
+  // Variablen auf Worker-Ebene deklarieren (Verhindert ReferenceError in finally)
   let scrapedPrice = null;
   let roomSizeM2 = null;
   let hotelDetails = { name: null, rating: null, reviewCount: null, address: null, image: null };
+  let domData = { dNights: 1, dPersons: 2, roomSize: null };
+  let extractedReviews = [];
   let browser = null;
 
-  // 1. Fallback: Nächte aus URL-Checkin/Checkout berechnen
-  const urlObj = new URL(url);
-  const checkin = urlObj.searchParams.get('checkin');
-  const checkout = urlObj.searchParams.get('checkout');
-  const adultsParam = parseInt(urlObj.searchParams.get('group_adults') || '0', 10);
-
-  let nightsFromUrl = null;
-  if (checkin && checkout) {
-    const diffTime = Math.abs(new Date(checkout) - new Date(checkin));
-    nightsFromUrl = Math.ceil(diffTime / (1000 * 60 * 60 * 24));
-  }
-
   try {
     browser = await chromium.launch({ headless: true });
     const context = await browser.newContext({
       userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
-      viewport: { width: 1280, height: 800 },
+      viewport: { width: 1280, height: 900 },
       locale: 'de-DE',
       timezoneId: 'Europe/Berlin'
     });
 
     const page = await context.newPage();
-    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 35000 });
-    await page.waitForTimeout(3000);
 
-    // 2. Hotel Details aus JSON-LD
+    let targetUrl = url;
+    if (!targetUrl.includes('#tab-reviews')) {
+      targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews';
+    }
+    if (!targetUrl.includes('sort_by=')) {
+      targetUrl += (targetUrl.includes('?') ? '&' : '?') + 'sort_by=f_recent_desc';
+    }
+
+    await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 35000 });
+    await page.waitForTimeout(2500);
+
+    // Klick auf Bewertungs-Reiter erzwingen
+    try {
+      const reviewTab = await page.$('#js--hp-gallery-scorecard a, a[href*="#tab-reviews"], [data-testid="review-score-link"]');
+      if (reviewTab) {
+        await reviewTab.click();
+        await page.waitForTimeout(2000);
+      }
+    } catch (e) {}
+
+    // Scrollen
+    await page.evaluate(() => {
+      const el = document.querySelector('#reviewListSorters') || document.querySelector('#tab-reviews');
+      if (el) el.scrollIntoView();
+      else window.scrollBy(0, 1000);
+    });
+    await page.waitForTimeout(2500);
+
+    // 1. JSON-LD Stammdaten
     hotelDetails = await page.evaluate(() => {
       const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
       for (const script of scripts) {
@@ -52,94 +87,178 @@ const worker = new Worker('scrape-jobs', async (job) => {
       return { name: null, rating: null, reviewCount: null, address: null, image: null };
     });
 
-    // 3. Nächte, Personen & Zimmergröße direkt aus den Booking-DOM-Selektoren lesen
+    // 2. Preis & DOM-Daten
-    const domData = await page.evaluate(() => {
+    domData = await page.evaluate(() => {
       let dNights = null;
       let dPersons = null;
-
+      const targetSelectors = ['.bui-price-display__label', '#searchbox-horizontal-occupancy-label', '[data-testid="price-for-x-nights"]'];
-      // Gezielte Suche in den von dir genannten Selektoren
-      const targetSelectors = [
-        '.bui-price-display__label',
-        '#searchbox-horizontal-occupancy-label',
-        '[data-testid="price-for-x-nights"]',
-        '#group_recommendation .bui-price-display__label',
-        '[data-testid="occupancy-config"]'
-      ];
-
       let combinedText = '';
-      for (const sel of targetSelectors) {
+      targetSelectors.forEach(s => document.querySelectorAll(s).forEach(el => combinedText += ' ' + el.innerText));
-        const els = document.querySelectorAll(sel);
+      if (!combinedText.trim()) combinedText = document.body.innerText;
-        els.forEach(el => {
-          if (el && el.innerText) combinedText += ' ' + el.innerText;
-        });
-      }
 
-      // Falls Selektoren leer sind, gesamten Text als Fallback nutzen
+      const nMatch = combinedText.match(/(\d+)\s*Nächte?/i);
-      if (!combinedText.trim()) {
+      if (nMatch) dNights = parseInt(nMatch[1], 10);
-        combinedText = document.body.innerText;
+      const pMatch = combinedText.match(/(\d+)\s*(?:Erwachsene|Erwachsener|Gäste|Personen)/i);
-      }
+      if (pMatch) dPersons = parseInt(pMatch[1], 10);
+      const rmMatch = document.body.innerText.match(/(\d+)\s*m²/i);
 
-      // Nächte matchen (z. B. "7 Nächte", "1 Nacht", "für 2 Nächte")
+      return { dNights, dPersons, roomSize: rmMatch ? parseInt(rmMatch[1], 10) : null };
-      const nightMatch = combinedText.match(/(\d+)\s*Nächte?/i);
-      if (nightMatch) dNights = parseInt(nightMatch[1], 10);
-
-      // Personen matchen (z. B. "2 Erwachsene", "1 Erwachsener", "2 Gäste")
-      const personMatch = combinedText.match(/(\d+)\s*(?:Erwachsene|Erwachsener|Gäste|Personen)/i);
-      if (personMatch) dPersons = parseInt(personMatch[1], 10);
-
-      // Zimmergröße (m²)
-      const roomMatch = document.body.innerText.match(/(\d+)\s*m²/i);
-      const roomSize = roomMatch ? parseInt(roomMatch[1], 10) : null;
-
-      return { dNights, dPersons, roomSize };
     });
 
     roomSizeM2 = domData.roomSize;
 
-    // Finale Werte bestimmen (Priorität: URL -> DOM -> Standard-Fallback)
+    const priceSelectors = ['[data-testid="price-and-discounted-price"]', '.totalPrice .bui-price-display__value', '.hprt-price-price'];
-    const finalNights = nightsFromUrl || domData.dNights || 1;
+    for (const sel of priceSelectors) {
-    const finalPersons = adultsParam || domData.dPersons || 2;
+      const el = await page.$(sel);
-
-    // 4. Preis extrahieren
-    const priceSelectors = [
-      '[data-testid="price-and-discounted-price"]',
-      '.totalPrice .bui-price-display__value',
-      '.hprt-price-price'
-    ];
-
-    for (const selector of priceSelectors) {
-      const el = await page.$(selector);
       if (el) {
         const text = await el.innerText();
-        if (text && /\d/.test(text)) {
+        if (text && /\d/.test(text)) { scrapedPrice = text.trim(); break; }
-          scrapedPrice = text.trim();
+      }
+    }
+
+    // 3. KOMMENTARE MIT PAGINATION BIS ZU 80 STÜCK EXTRAHIEREN
+    let pageCount = 0;
+    const seenKeys = new Set();
+
+    while (extractedReviews.length < 80 && pageCount < 15) {
+      const pageReviews = await page.evaluate(({ westernCountries }) => {
+        const dateElements = Array.from(document.querySelectorAll('[data-testid="review-date"]'));
+        const results = [];
+        const currentYear = 2026;
+
+        for (const dateEl of dateElements) {
+          // Navigiere hoch zum Container, der SOWOHL Person/Land ALS AUCH Bewertungstext enthält
+          let card = dateEl.parentElement;
+          while (card && card !== document.body) {
+            if (card.querySelector('[data-testid="review-avatar"]') || card.querySelector('[aria-label="Bewertende Person"]')) {
+              break;
+            }
+            card = card.parentElement;
+          }
+          if (!card || card === document.body) {
+            card = dateEl.closest('li') || dateEl.parentElement.parentElement.parentElement;
+          }
+
+          const cardText = card ? card.innerText : '';
+          if (!cardText) continue;
+
+          // Herkunftsland prüfen
+          const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase()));
+          if (!isWestern) continue;
+
+          // Datum prüfen (<= 2 Jahre)
+          const dateText = dateEl.innerText || '';
+          const yearMatch = cardText.match(/\b(20\d{2})\b/);
+          if (yearMatch) {
+            const reviewYear = parseInt(yearMatch[1], 10);
+            if (currentYear - reviewYear > 2) continue;
+          }
+
+          const titleEl = card.querySelector('[data-testid="review-title"]');
+          const posEl = card.querySelector('[data-testid="review-positive-text"], .review_pos');
+          const negEl = card.querySelector('[data-testid="review-negative-text"], .review_neg');
+
+          const title = titleEl ? titleEl.innerText.trim() : '';
+          const positive = posEl ? posEl.innerText.trim() : '';
+          const negative = negEl ? negEl.innerText.trim() : '';
+
+          let generalComment = '';
+          if (!positive && !negative) {
+            const commentEl = card.querySelector('.b99b6ef58f, .a17eaf5728');
+            if (commentEl && !commentEl.innerText.includes('keinen Kommentar')) {
+              generalComment = commentEl.innerText.trim();
+            }
+          }
+
+          if (title || positive || negative || generalComment) {
+            results.push({
+              title,
+              positive: positive || generalComment,
+              negative,
+              date: dateText
+            });
+          }
+        }
+
+        return results;
+      }, { westernCountries: WESTERN_COUNTRIES });
+
+      // Duplikate vermeiden
+      for (const rev of pageReviews) {
+        const key = `${rev.title}-${rev.positive}-${rev.date}`;
+        if (!seenKeys.has(key) && extractedReviews.length < 80) {
+          seenKeys.add(key);
+          extractedReviews.push(rev);
+        }
+      }
+
+      if (extractedReviews.length >= 80) break;
+
+      // Klick auf "Nächste Seite" Pfeil
+      try {
+        const nextBtn = await page.$(
+          '#reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"], [data-testid="pagination-next-page"]'
+        );
+
+        if (nextBtn) {
+          const isDisabled = await nextBtn.getAttribute('disabled');
+          if (isDisabled !== null) break;
+
+          await nextBtn.click();
+          await page.waitForTimeout(2500);
+          pageCount++;
+        } else {
+          break;
+        }
+      } catch (e) {
         break;
       }
     }
-    }
 
-    // 5. Webhook mit exakten Daten absenden
+    console.log(`[Job ${job.id}] ${extractedReviews.length} Kommentare über ${pageCount + 1} Seiten extrahiert.`);
-    if (webhookUrl && webhookUrl.startsWith('http')) {
+
-      await axios.post(webhookUrl, {
+  } catch (error) {
-        jobId: job.id,
+    console.error(`[Job ${job.id}] Fehler beim Scraping:`, error.message);
-        url,
+  } finally {
-        hotelName: hotelDetails.name,
+    if (browser) await browser.close();
+
+    // 1. PREISHISTORIE IN POSTGRES SPEICHERN
+    if (appWebhookUrl) {
+      try {
+        await axios.post(appWebhookUrl, {
           price: scrapedPrice,
           roomSizeM2,
+          hotelName: hotelDetails.name,
           rating: hotelDetails.rating,
           reviewCount: hotelDetails.reviewCount,
           address: hotelDetails.address,
           image: hotelDetails.image,
-        nights: finalNights,
+          nights: domData.dNights || 1,
-        persons: finalPersons,
+          persons: domData.dPersons || 2,
-        timestamp: new Date().toISOString()
         });
+        console.log(`[Job ${job.id}] ✅ Preishistorie direkt in Postgres gespeichert!`);
+      } catch (e) {
+        console.error(`[Job ${job.id}] ❌ Fehler beim Speichern der Preishistorie:`, e.message);
+      }
     }
 
-  } catch (error) {
+    // 2. REVIEWS AN N8N SENDEN
-    console.error(`[Job ${job.id}] Fehler:`, error.message);
+    if (webhookUrl) {
-  } finally {
+      try {
-    if (browser) await browser.close();
+        await axios.post(webhookUrl, {
+          jobId: job.id,
+          hotelId,
+          hotelName: hotelDetails.name,
+          reviews: extractedReviews,
+          timestamp: new Date().toISOString()
+        });
+        console.log(`[Job ${job.id}] 🚀 ${extractedReviews.length} Reviews an n8n gesendet!`);
+      } catch (e) {
+        console.error(`[Job ${job.id}] ❌ Fehler beim Senden an n8n:`, e.message);
+      }
+    }
   }
 
 }, { ...REDIS_CONFIG, concurrency: 2 });
+
+console.log('🚀 Scraper Worker bereit...');