fix(scraper): perfect review dialog opening, pagination, and multi-page review extraction

src/index.js

@@ -23,7 +23,10 @@ const WESTERN_COUNTRIES = [
   'United Kingdom', 'UK', 'Niederlande', 'Netherlands', 'Frankreich', 'France',
   'Spanien', 'Spain', 'Italien', 'Italy', 'Kanada', 'Canada', 'Australien', 'Australia',
   'Belgien', 'Belgium', 'Schweden', 'Sweden', 'Norwegen', 'Norway', 'Dänemark', 'Denmark',
-  'Finnland', 'Finland', 'Irland', 'Ireland', 'Luxemburg', 'Luxembourg', 'Indien', 'India'
+  'Finnland', 'Finland', 'Irland', 'Ireland', 'Luxemburg', 'Luxembourg', 'Indien', 'India',
+  'Singapur', 'Singapore', 'Malaysia', 'Sri Lanka', 'Neuseeland', 'New Zealand',
+  'Südafrika', 'South Africa', 'Vereinigte Arabische Emirate', 'UAE', 'Dubai',
+  'Polen', 'Poland', 'Tschechien', 'Czech', 'Japan', 'Korea'
 ];
 
 // ==========================================
@@ -163,8 +166,8 @@ async function safeEvaluate(page, fn, fallback = null, arg = undefined) {
 // BULLMQ WORKER: SCRAPING & REVIEWS
 // ==========================================
 const worker = new Worker('scrape-jobs', async (job) => {
-  const { url, webhookUrl, appWebhookUrl, hotelId, debug } = job.data;
-  const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http'));
+  const { url, webhookUrl, appWebhookUrl, hotelId, debug, mode } = job.data;
+  const isReviewScan = Boolean((webhookUrl && webhookUrl.startsWith('http')) || mode === 'reviews' || mode === 'full');
   const isDebugMode = Boolean(debug || DEBUG_HTML);
   
   console.log(`\n==================================================`);
@@ -187,29 +190,21 @@ const worker = new Worker('scrape-jobs', async (job) => {
       args: [
         '--no-sandbox',
         '--disable-setuid-sandbox',
+        '--disable-dev-shm-usage',
         '--disable-blink-features=AutomationControlled',
-        '--disable-infobars'
+        '--window-size=1920,1080'
       ]
     });
 
     const context = await browser.newContext({
-      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
-      viewport: { width: 1366, height: 900 },
+      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
+      viewport: { width: 1920, height: 1080 },
       locale: 'de-DE',
-      timezoneId: 'Europe/Berlin',
-      extraHTTPHeaders: {
-        'Accept-Language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
-        'Upgrade-Insecure-Requests': '1',
-        'Sec-Ch-Ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"',
-        'Sec-Ch-Ua-Mobile': '?0',
-        'Sec-Ch-Ua-Platform': '"Windows"'
-      }
+      timezoneId: 'Europe/Berlin'
     });
 
-    const page = await context.newPage();
-
     // Verhindert Erkennung über navigator.webdriver und emuliert Standard-Browser-Eigenschaften
-    await page.addInitScript(() => {
+    await context.addInitScript(() => {
       Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
       window.chrome = {
         runtime: {},
@@ -225,6 +220,8 @@ const worker = new Worker('scrape-jobs', async (job) => {
       });
     });
 
+    const page = await context.newPage();
+
     let targetUrl = cleanBookingUrl(url);
 
     console.log(`[Job ${job.id}] 🌐 Bereinigte Ziel-URL: ${targetUrl}`);
@@ -289,20 +286,12 @@ const worker = new Worker('scrape-jobs', async (job) => {
 
     // Cookie-Banner & störende Overlays für alle Scans sofort schließen
     try {
-      const dismissSelectors = [
-        '#onetrust-accept-btn-handler',
-        'button#onetrust-reject-all-handler',
-        'button[data-testid="selection-modal-close-btn"]',
-        'button[aria-label="Schließen"]',
-        'button[aria-label="Close"]',
-        'button.modal-mask-closeBtn'
-      ];
-      for (const btnSel of dismissSelectors) {
-        const btn = await page.$(btnSel);
-        if (btn) {
-          await btn.click().catch(() => {});
-          await page.waitForTimeout(300);
-        }
+      const dismissBtns = await page.$$(
+        'button[aria-label*="schließen"], button[aria-label*="Schließen"], button[aria-label*="Close"], button[aria-label*="Dismiss"], #onetrust-accept-btn-handler, button:has-text("Akzeptieren"), button:has-text("Alle akzeptieren"), button[data-testid="selection-modal-close-btn"]'
+      );
+      for (const btn of dismissBtns) {
+        await btn.click().catch(() => {});
+        await page.waitForTimeout(300);
       }
     } catch (e) {}
 
@@ -579,67 +568,35 @@ const worker = new Worker('scrape-jobs', async (job) => {
     if (isReviewScan) {
       console.log(`[Job ${job.id}] 🚀 Starte gezielte Review-Extraction für "${hotelDetails.name || 'Hotel'}"...`);
 
-      // A. Cookie-Banner und störende Overlays schließen
-      try {
-        const dismissSelectors = [
-          '#onetrust-accept-btn-handler',
-          'button#onetrust-reject-all-handler',
-          'button[data-testid="selection-modal-close-btn"]',
-          'button[aria-label="Schließen"]',
-          'button[aria-label="Close"]'
-        ];
-        for (const btnSel of dismissSelectors) {
-          const btn = await page.$(btnSel);
-          if (btn) {
-            await btn.click().catch(() => {});
-            await page.waitForTimeout(400);
-          }
-        }
-      } catch (e) {}
-
       // B. Herunterscrollen um Lazy-Components zu hydrieren und Review-Dialog zu öffnen
       console.log(`[Job ${job.id}] 📜 Scrolle herunter, um verzögert geladene Review-Komponenten zu aktivieren...`);
       await page.evaluate(() => window.scrollTo(0, 4000));
-      await page.waitForTimeout(2000);
+      await page.waitForTimeout(2500);
 
       // C. Klick auf Review-Trigger ausführen
-      let isPanelOpen = false;
-      try {
-        console.log(`[Job ${job.id}] 👉 Öffne Review-Dialog über 'Alle Bewertungen lesen' / 'Gästebewertungen'...`);
-        isPanelOpen = await page.evaluate(() => {
-          const btn = document.querySelector('button[data-testid="fr-read-all-reviews"]') ||
-                      document.querySelector('button[data-testid="review-score-read-all-actionable"]') ||
-                      document.getElementById('reviews-tab-trigger') ||
-                      document.querySelector('[data-testid="Property-Header-Nav-Tab-Trigger-reviews"]');
-          if (btn) {
-            btn.scrollIntoView({ behavior: 'instant', block: 'center' });
-            btn.click();
-            btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
-            return true;
-          }
-          return false;
-        }).catch(() => false);
-
-        // Warten bis das Dialog-Panel mit den 1.000+ Bewertungen gerendert ist
-        try {
-          await page.waitForSelector(
-            'div[role="dialog"] [data-testid="review-card"], #reviewCardsSection [data-testid="review-card"], [data-testid="review-card"]',
-            { timeout: 15000 }
-          );
-          console.log(`[Job ${job.id}] 🌟 Review-Dialog ('role="dialog"') mit allen Bewertungen erfolgreich geladen!`);
-        } catch (waitErr) {
-          console.log(`[Job ${job.id}] ⚠️ Dialog-Karten nicht sofort erkannt, fahre mit vorhandenem DOM fort...`);
+      console.log(`[Job ${job.id}] 👉 Öffne Review-Dialog über 'Alle Bewertungen lesen' / 'Gästebewertungen'...`);
+      await page.evaluate(() => {
+        const btn = document.querySelector('button[data-testid="fr-read-all-reviews"]') ||
+                    document.querySelector('button[data-testid="review-score-read-all-actionable"]') ||
+                    document.getElementById('reviews-tab-trigger') ||
+                    document.querySelector('[data-testid="Property-Header-Nav-Tab-Trigger-reviews"]');
+        if (btn) {
+          btn.scrollIntoView({ behavior: 'instant', block: 'center' });
+          btn.click();
+          btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
+          return true;
         }
+        return false;
+      }).catch(() => false);
 
-        // Screenshot zur visuellen Kontrolle aktualisieren
-        try {
-          await page.screenshot({ path: 'src/review-debug.png', fullPage: false });
-          console.log(`[Job ${job.id}] 📸 Screenshot aktualisiert unter: scraper-service/src/review-debug.png`);
-        } catch (e) {}
+      // 4 Sekunden warten bis das Dialog-Panel mit den 1.000+ Bewertungen vollständig geöffnet ist
+      await page.waitForTimeout(4000);
 
-      } catch (e) {
-        console.log(`[Job ${job.id}] ⚠️ Fehler beim Klick auf Review-Trigger:`, e.message);
-      }
+      // Screenshot zur visuellen Kontrolle aktualisieren
+      try {
+        await page.screenshot({ path: 'src/review-debug.png', fullPage: false });
+        console.log(`[Job ${job.id}] 📸 Screenshot aktualisiert unter: scraper-service/src/review-debug.png`);
+      } catch (e) {}
 
       // D. Sortierung auf "Neueste zuerst" (NEWEST_FIRST) umstellen
       try {
@@ -660,84 +617,36 @@ const worker = new Worker('scrape-jobs', async (job) => {
         pageCount++;
         console.log(`[Job ${job.id}] 📄 Scanne Bewertungsseite ${pageCount} (Bisher gesammelt: ${extractedReviews.length})...`);
 
-        const evalResult = await safeEvaluate(page, ({ westernCountries }) => {
-          let cards = Array.from(document.querySelectorAll([
-            '#reviewCardsSection [data-testid="review-card"]',
-            'div[role="dialog"] [data-testid="review-card"]',
-            '[data-testid="review-card"]',
-            '[data-testid="review-cards"] > li',
-            '[data-testid="review-cards"] > div',
-            'div[data-testid*="review-card"]',
-            'li[data-testid*="review-card"]',
-            'li.review_list_new_item_block',
-            'div.c-review-block__item',
-            'div[itemprop="review"]',
-            'li.review_item',
-            'div.c-review',
-            'div.review_item',
-            '.c-review-block',
-            'li.c-review-block'
-          ].join(',')));
-
-          if (cards.length === 0) {
-            const container = document.querySelector('#hp-reviews-sliding, #reviewCardsSection, [data-testid="review-cards"], div[role="dialog"]');
-            if (container) {
-              cards = Array.from(container.querySelectorAll('li, div.c-review-block, div.c-review')).filter(el => (el.innerText || '').length > 30);
-            }
-          }
-
-          // Filterboxen und Scorecard-Zusammenfassungen explizit ausschließen
-          cards = cards.filter(el => {
-            const text = el.innerText || '';
-            return text.length > 20 &&
-                   !text.includes('Wählen Sie Themen') && 
-                   !text.includes('Kategorien:') &&
-                   !el.classList.contains('review_score_breakdown');
-          });
-
-          const results = [];
-          const debugLogs = [];
+        // Extrahiere alle Review-Karten aus dem Dialog / DOM
+        const pageReviews = await page.evaluate(() => {
+          const cards = Array.from(document.querySelectorAll(
+            '#reviewCardsSection [data-testid="review-card"], div[role="dialog"] [data-testid="review-card"], [data-testid="review-card"]'
+          ));
+          const items = [];
           const currentYear = 2026;
 
-          for (let idx = 0; idx < cards.length; idx++) {
-            const card = cards[idx];
-            const cardText = card.innerText || '';
-            if (!cardText.trim()) continue;
-
-            // 1. Herkunftsland des Gastes ermitteln
+          for (const card of cards) {
+            const titleEl = card.querySelector('[data-testid="review-title"], h4');
+            const posEl = card.querySelector('[data-testid="review-positive-text"]');
+            const negEl = card.querySelector('[data-testid="review-negative-text"]');
+            const dateEl = card.querySelector('[data-testid="review-date"]');
             const flagImg = card.querySelector('[data-testid="review-avatar"] .fff1944c52 img[alt], [data-testid="review-country"] img[alt], img.b8d1620349');
             const countrySpan = card.querySelector('[data-testid="review-avatar"] .fff1944c52 span:not(.cec4c9cfef), [data-testid="review-country"]');
-            const guestCountryText = (flagImg ? flagImg.getAttribute('alt') : (countrySpan ? countrySpan.innerText : '')).trim();
-
-            if (guestCountryText && guestCountryText.length > 1) {
-              const isWestern = westernCountries.some(c => guestCountryText.toLowerCase().includes(c.toLowerCase()));
-              if (!isWestern) {
-                const preview = guestCountryText.substring(0, 50).replace(/\n/g, ' ');
-                debugLogs.push(`Karte #${idx + 1}: ❌ Land nicht in Liste ("${preview}")`);
-                continue;
-              }
-            }
-
-            // 2. Datum / Jahr prüfen (letzte 2 Jahre)
-            const yearMatch = cardText.match(/\b(20\d{2})\b/);
-            if (yearMatch) {
-              const reviewYear = parseInt(yearMatch[1], 10);
-              if (currentYear - reviewYear > 2) {
-                debugLogs.push(`Karte #${idx + 1}: ❌ Zu alt (${reviewYear})`);
-                continue;
-              }
-            }
-
-            // 3. Texte extrahieren
-            const titleEl = card.querySelector('[data-testid="review-title"], .c-review__title, h4, .review_title');
-            const posEl = card.querySelector('[data-testid="review-positive-text"], .c-review__row--positive, .review_pos');
-            const negEl = card.querySelector('[data-testid="review-negative-text"], .c-review__row--negative, .review_neg');
-            const dateEl = card.querySelector('[data-testid="review-date"], .c-review__date, .review_item_date');
 
             const title = titleEl ? titleEl.innerText.trim() : '';
             const positive = posEl ? posEl.innerText.trim().replace(/^„|“$/g, '').trim() : '';
             const negative = negEl ? negEl.innerText.trim().replace(/^„|“$/g, '').trim() : '';
-            const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell');
+            const dateText = dateEl ? dateEl.innerText.trim() : 'Aktuell';
+            const country = (flagImg ? flagImg.getAttribute('alt') : (countrySpan ? countrySpan.innerText : '')).trim();
+
+            const cardText = card.innerText || '';
+            const yearMatch = cardText.match(/\b(20\d{2})\b/);
+            if (yearMatch) {
+              const reviewYear = parseInt(yearMatch[1], 10);
+              if (currentYear - reviewYear > 2) {
+                continue;
+              }
+            }
 
             let fallbackText = '';
             if (!positive && !negative) {
@@ -745,47 +654,50 @@ const worker = new Worker('scrape-jobs', async (job) => {
             }
 
             if (title || positive || negative || fallbackText) {
-              results.push({ title, positive: positive || fallbackText, negative, date: dateText });
-              debugLogs.push(`Karte #${idx + 1}: ✅ Akzeptiert ("${title || (positive ? positive.substring(0, 25) : 'Review')}")`);
+              items.push({ title, positive: positive || fallbackText, negative, date: dateText, country });
             }
           }
+          return items;
+        }).catch(err => {
+          console.error(`[Job ${job.id}] Evaluate error on page ${pageCount}:`, err.message);
+          return [];
+        });
 
-          return {
-            totalCards: cards.length,
-            results,
-            debugLogs: debugLogs.slice(0, 8)
-          };
-        }, { totalCards: 0, results: [], debugLogs: [] }, { westernCountries: WESTERN_COUNTRIES });
+        console.log(`[Job ${job.id}] 🔍 Echte Gästekarten auf Seite ${pageCount} gefunden: ${pageReviews.length}`);
 
-        console.log(`[Job ${job.id}] 🔍 Echte Gästekarten gefunden: ${evalResult.totalCards}`);
-        if (evalResult.debugLogs) {
-          evalResult.debugLogs.forEach(log => console.log(`[Job ${job.id}]   └─ ${log}`));
-        }
-
-        // Deduplizierung
-        for (const rev of evalResult.results) {
+        for (const rev of pageReviews) {
+          if (rev.country) {
+            const isWestern = WESTERN_COUNTRIES.some(c => rev.country.toLowerCase().includes(c.toLowerCase()));
+            if (!isWestern) {
+              console.log(`[Job ${job.id}]   └─ ❌ Land nicht in Liste: "${rev.country}"`);
+              continue;
+            }
+          }
           const key = `${rev.title}-${rev.positive}-${rev.date}`;
           if (!seenKeys.has(key) && extractedReviews.length < 80) {
             seenKeys.add(key);
             extractedReviews.push(rev);
+            console.log(`[Job ${job.id}]   └─ ✅ Akzeptiert: "${rev.title || rev.positive.substring(0, 30)}" (${rev.country || 'International'})`);
           }
         }
 
         if (extractedReviews.length >= 80) break;
 
-        // Weiterblättern im Modal oder der Hauptseite
-        try {
-          const nextBtn = await page.$(
-            'div[role="dialog"] button[aria-label="Nächste Seite"], button[aria-label="Nächste Seite"], button[aria-label*="Nächste Seite"], #hp-reviews-sliding button[aria-label*="Nächste"], button[aria-label*="Next"], #reviewCardsSection button:has(svg), a.paging-next, li.bui-pagination__next-arrow a'
-          );
-          if (nextBtn) {
-            const isDisabled = await nextBtn.getAttribute('disabled');
-            if (isDisabled === null) {
-              await nextBtn.click({ force: true });
-              await page.waitForTimeout(2500);
-            } else break;
-          } else break;
-        } catch (e) {
+        // Weiterblättern im Modal
+        const nextResult = await page.evaluate(() => {
+          const nextBtn = document.querySelector('div[role="dialog"] button[aria-label="Nächste Seite"], button[aria-label="Nächste Seite"], button[aria-label*="Nächste Seite"]');
+          if (nextBtn && !nextBtn.disabled && !nextBtn.hasAttribute('disabled')) {
+            nextBtn.scrollIntoView({ behavior: 'instant', block: 'center' });
+            nextBtn.click();
+            nextBtn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
+            return true;
+          }
+          return false;
+        }).catch(() => false);
+
+        if (nextResult) {
+          await page.waitForTimeout(3000);
+        } else {
           break;
         }
       }