fix(scraper): enable reliable multi-page review dialog extraction with lazy-scroll and exact country parsing

.gitignore

@@ -11,3 +11,4 @@ src/*.png
 debug/
 
 .DS_Store
+

AGENTS.md

@@ -0,0 +1,21 @@
+# Engineering & Scraping Guidelines
+
+## 1. Git Commit & Repository Hygiene
+- **Conventional Commits:** Write clear, concise commit messages prefixed with standard types (e.g., `feat:`, `fix:`, `refactor:`, `docs:`).
+- **Exclude Transient Artifacts:** Never stage or commit generated debug files (e.g., `src/*.html`, `src/*.png`, large logs) to version control. Always maintain `.gitignore` to keep git packs lean and prevent remote unpacker errors on Gitea/GitHub.
+
+## 2. Web Scraping & DOM Inspection Strategy
+- **Rendered DOM Dumping for Debugging:** When debugging extraction failures on dynamic or anti-bot protected pages:
+  - Capture the post-render HTML via `await page.content()` and save it to `src/hotel-debug.html`.
+  - Capture a synchronized screenshot (`src/hotel-debug.png`).
+  - Inspect the raw rendered DOM to verify if the page was served as standard SSR, an A/B test SPA variant, or intercepted by a challenge (e.g. AWS WAF).
+- **Multi-Tier Extraction Hierarchy:** Always implement fallbacks in this order:
+  1. **Deep JSON-LD Parsing:** Traverse all `<script type="application/ld+json">` tags, including `@graph` arrays and varied accommodation schemas (`Hotel`, `Resort`, `Apartment`, `Hostel`, `LodgingBusiness`, etc.).
+  2. **DOM Selectors:** Fallback to standard test IDs, classes, and element hierarchy (`data-testid`, header titles, score components).
+  3. **Meta Tags:** Fallback to OpenGraph and Twitter meta tags (`og:title`, `og:image`, `og:description`).
+  4. **URL Slug Extraction:** As an ultimate safety net, derive clean human-readable names from the URL path if the HTML body was obstructed.
+- **Anti-Bot & WAF Handling:**
+  - Sanitize input URLs by stripping transient tokens (`chal_t`, `force_referer`, session IDs).
+  - Emulate full browser properties (`window.chrome`, `navigator.plugins`, `navigator.languages`).
+  - Actively detect WAF challenge interstitial pages and wait for client-side JavaScript token computation and automatic redirection before evaluating the DOM.
+

src/index.js

@@ -139,6 +139,26 @@ function cleanBookingUrl(rawUrl) {
   }
 }
 
+// ==========================================
+// HILFSFUNKTION: SICHERE EVALUATION (RESISTENT GEGEN REDIRECTS)
+// ==========================================
+async function safeEvaluate(page, fn, fallback = null, arg = undefined) {
+  try {
+    return arg !== undefined ? await page.evaluate(fn, arg) : await page.evaluate(fn);
+  } catch (err) {
+    if (err.message && (err.message.includes('Execution context was destroyed') || err.message.includes('navigating') || err.message.includes('Target closed'))) {
+      await page.waitForLoadState('domcontentloaded').catch(() => {});
+      await page.waitForTimeout(2500);
+      try {
+        return arg !== undefined ? await page.evaluate(fn, arg) : await page.evaluate(fn);
+      } catch (innerErr) {
+        return fallback;
+      }
+    }
+    return fallback;
+  }
+}
+
 // ==========================================
 // BULLMQ WORKER: SCRAPING & REVIEWS
 // ==========================================
@@ -206,17 +226,17 @@ const worker = new Worker('scrape-jobs', async (job) => {
     });
 
     let targetUrl = cleanBookingUrl(url);
-    if (isReviewScan && !targetUrl.includes('#tab-reviews')) {
-      targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews';
-    }
 
     console.log(`[Job ${job.id}] 🌐 Bereinigte Ziel-URL: ${targetUrl}`);
 
     await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 45000 });
+    await page.waitForLoadState('domcontentloaded').catch(() => {});
     await page.waitForTimeout(2000);
 
-    // Prüfen, ob AWS WAF Challenge aktiv ist
+    // Prüfen, ob AWS WAF Challenge aktiv ist (sicher gegenüber gleichzeitiger Navigation)
-    const isWafChallenge = await page.evaluate(() => {
+    let isWafChallenge = false;
+    try {
+      isWafChallenge = await page.evaluate(() => {
         return Boolean(
           document.querySelector('#challenge-container') ||
           document.querySelector('script[src*="challenge.js"]') ||
@@ -224,6 +244,21 @@ const worker = new Worker('scrape-jobs', async (job) => {
           (document.title === '' && (document.body ? document.body.innerText.includes('verify that you') : false))
         );
       });
+    } catch (navErr) {
+      await page.waitForLoadState('domcontentloaded').catch(() => {});
+      await page.waitForTimeout(2500);
+      try {
+        isWafChallenge = await page.evaluate(() => {
+          return Boolean(
+            document.querySelector('#challenge-container') ||
+            document.querySelector('script[src*="challenge.js"]') ||
+            window.AwsWafIntegration
+          );
+        });
+      } catch (e) {
+        isWafChallenge = false;
+      }
+    }
 
     if (isWafChallenge) {
       console.log(`[Job ${job.id}] 🛡️ AWS WAF Challenge erkannt. Warte auf automatische Token-Weiterleitung...`);
@@ -234,12 +269,14 @@ const worker = new Worker('scrape-jobs', async (job) => {
           return !hasChallenge || Boolean(hasContent);
         }, { timeout: 25000 });
         console.log(`[Job ${job.id}] 🔓 Weiterleitung nach AWS WAF erfolgt!`);
+        await page.waitForLoadState('domcontentloaded').catch(() => {});
         await page.waitForTimeout(2000);
       } catch (wafErr) {
         console.log(`[Job ${job.id}] ⚠️ AWS WAF Timeout. Versuche direkten Reload der Basis-URL...`);
         try {
           const parsed = new URL(targetUrl);
           await page.goto(parsed.origin + parsed.pathname, { waitUntil: 'domcontentloaded', timeout: 30000 });
+          await page.waitForLoadState('domcontentloaded').catch(() => {});
           await page.waitForTimeout(3000);
         } catch (reloadErr) {}
       }
@@ -290,7 +327,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
     } catch (e) {}
 
     // 2. STAMMDATEN EXTRAHIEREN (JSON-LD + DOM + META FALLBACKS)
-    hotelDetails = await page.evaluate(() => {
+    hotelDetails = await safeEvaluate(page, () => {
       let name = null;
       let rating = null;
       let reviewCount = null;
@@ -450,7 +487,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
       }
 
       return { name, rating, reviewCount, address, image };
-    });
+    }, { name: null, rating: null, reviewCount: null, address: null, image: null });
 
     // Fallback: Name aus URL-Slug ableiten wenn im HTML nichts gefunden wurde
     if (!hotelDetails.name) {
@@ -458,7 +495,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
     }
 
     // 3. DOM-DATEN (Nächte, Personen, Zimmergröße)
-    const extraDom = await page.evaluate(() => {
+    const extraDom = await safeEvaluate(page, () => {
       try {
         const bodyText = document.body ? (document.body.innerText || '') : '';
         const nMatch = bodyText.match(/(\d+)\s*Nächte?/i);
@@ -472,14 +509,14 @@ const worker = new Worker('scrape-jobs', async (job) => {
       } catch (e) {
         return { dNights: 1, dPersons: 2, roomSize: null };
       }
-    });
+    }, { dNights: 1, dPersons: 2, roomSize: null });
 
     domData.dNights = extraDom.dNights;
     domData.dPersons = extraDom.dPersons;
     roomSizeM2 = extraDom.roomSize;
 
     // PREIS AUSLESEN
-    scrapedPrice = await page.evaluate(() => {
+    scrapedPrice = await safeEvaluate(page, () => {
       try {
         const selectors = [
           '[data-testid="price-and-discounted-price"]',
@@ -502,10 +539,10 @@ const worker = new Worker('scrape-jobs', async (job) => {
         }
       } catch (e) {}
       return null;
-    });
+    }, null);
 
     // GENIUS BADGE PRÜFUNG
-    hasGenius = await page.evaluate(() => {
+    hasGenius = await safeEvaluate(page, () => {
       try {
         const geniusSelectors = [
           '.m-badge__genius',
@@ -519,7 +556,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
       } catch (e) {
         return false;
       }
-    });
+    }, false);
 
     console.log(`[Job ${job.id}] 📋 Extrahierte Stammdaten:`, {
       name: hotelDetails.name,
@@ -560,59 +597,38 @@ const worker = new Worker('scrape-jobs', async (job) => {
         }
       } catch (e) {}
 
-      // B. 500px nach unten scrollen, damit Bookings Sticky-Header (#reviews-tab-trigger) überhaupt sichtbar & aktiv wird
+      // B. Herunterscrollen um Lazy-Components zu hydrieren und Review-Dialog zu öffnen
-      await page.evaluate(() => window.scrollBy(0, 550));
+      console.log(`[Job ${job.id}] 📜 Scrolle herunter, um verzögert geladene Review-Komponenten zu aktivieren...`);
-      await page.waitForTimeout(1500);
+      await page.evaluate(() => window.scrollTo(0, 4000));
+      await page.waitForTimeout(2000);
 
-      // C. Klick per echtem Maus-Event auf den Trigger ausführen
+      // C. Klick auf Review-Trigger ausführen
-      let clicked = false;
+      let isPanelOpen = false;
       try {
-        // Liste möglicher Klick-Ziele (Header-Tab, Scorecard-Badge oder Text "Gästebewertungen")
+        console.log(`[Job ${job.id}] 👉 Öffne Review-Dialog über 'Alle Bewertungen lesen' / 'Gästebewertungen'...`);
-        const triggerLocators = [
+        isPanelOpen = await page.evaluate(() => {
-          page.locator('#reviews-tab-trigger').first(),
+          const btn = document.querySelector('button[data-testid="fr-read-all-reviews"]') ||
-          page.locator('[data-testid="Property-Header-Nav-Tab-Trigger-reviews"]').first(),
+                      document.querySelector('button[data-testid="review-score-read-all-actionable"]') ||
-          page.locator('a:has-text("Gästebewertungen")').first(),
+                      document.getElementById('reviews-tab-trigger') ||
-          page.locator('button:has-text("Gästebewertungen")').first(),
+                      document.querySelector('[data-testid="Property-Header-Nav-Tab-Trigger-reviews"]');
-          page.locator('[data-testid="review-score-right-component"]').first(),
+          if (btn) {
-          page.locator('[data-testid="review-score-component"]').first()
+            btn.scrollIntoView({ behavior: 'instant', block: 'center' });
-        ];
+            btn.click();
-
+            btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
-        for (const loc of triggerLocators) {
-          if (await loc.count() > 0 && await loc.isVisible()) {
-            console.log(`[Job ${job.id}] 👉 Sichtbaren Review-Trigger gefunden. Führe Klick aus...`);
-            
-            // Reale Maus-Koordinaten ermitteln & klicken
-            const box = await loc.boundingBox();
-            if (box) {
-              await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2);
-              clicked = true;
-              await page.waitForTimeout(3500);
-              break;
-            } else {
-              await loc.click({ force: true }).catch(() => {});
-              clicked = true;
-              await page.waitForTimeout(3500);
-              break;
-            }
-          }
-        }
-
-        // Fallback: JS-Klick direkt auf das HTML-Element mit Text "Gästebewertungen"
-        if (!clicked) {
-          console.log(`[Job ${job.id}] ⚠️ Fallback: Klick per DOM-Suche nach 'Gästebewertungen'...`);
-          clicked = await page.evaluate(() => {
-            const el = Array.from(document.querySelectorAll('a, button, span, div'))
-              .find(e => e.innerText && e.innerText.includes('Gästebewertungen') && (e.id === 'reviews-tab-trigger' || e.closest('#reviews-tab-trigger') || e.getAttribute('data-testid')?.includes('reviews')));
-            if (el) {
-              const clickTarget = el.closest('a') || el.closest('button') || el;
-              clickTarget.click();
-              clickTarget.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
             return true;
           }
           return false;
         }).catch(() => false);
 
-          if (clicked) await page.waitForTimeout(3500);
+        // Warten bis das Dialog-Panel mit den 1.000+ Bewertungen gerendert ist
+        try {
+          await page.waitForSelector(
+            'div[role="dialog"] [data-testid="review-card"], #reviewCardsSection [data-testid="review-card"], [data-testid="review-card"]',
+            { timeout: 15000 }
+          );
+          console.log(`[Job ${job.id}] 🌟 Review-Dialog ('role="dialog"') mit allen Bewertungen erfolgreich geladen!`);
+        } catch (waitErr) {
+          console.log(`[Job ${job.id}] ⚠️ Dialog-Karten nicht sofort erkannt, fahre mit vorhandenem DOM fort...`);
         }
 
         // Screenshot zur visuellen Kontrolle aktualisieren
@@ -625,40 +641,13 @@ const worker = new Worker('scrape-jobs', async (job) => {
         console.log(`[Job ${job.id}] ⚠️ Fehler beim Klick auf Review-Trigger:`, e.message);
       }
 
-      // Warten bis Review-Karten im Sliding Panel oder DOM sichtbar sind
+      // D. Sortierung auf "Neueste zuerst" (NEWEST_FIRST) umstellen
-      try {
-        await page.waitForSelector(
-          '[data-testid="review-card"], [data-testid="review-cards"], #hp-reviews-sliding, div[data-component="hotel/reviews/review-card"], li.review_item',
-          { timeout: 8000 }
-        );
-        console.log(`[Job ${job.id}] 👁️ Review-Bereich erfolgreich gerendert.`);
-      } catch (e) {
-        console.log(`[Job ${job.id}] ⚠️ Wartezeit abgelaufen – starte Auslesen aus dem aktuellen DOM...`);
-      }
-
-      // C. Sortierung auf "Neueste zuerst" (NEWEST_FIRST) umstellen
       try {
         const sorterLocator = page.locator('#reviewListSorters, [data-testid="reviews-sorter-component"], select[name="reviewListSorters"]').first();
         if (await sorterLocator.count() > 0) {
           console.log(`[Job ${job.id}] 🔄 Stelle Sortierung auf 'NEWEST_FIRST' (Neueste zuerst) um...`);
           await sorterLocator.selectOption('NEWEST_FIRST');
-          await page.waitForTimeout(2500); // Warten bis Booking die Liste per AJAX neu lädt
+          await page.waitForTimeout(3000); // Warten bis Booking die Liste per AJAX neu lädt
-        } else {
-          // Fallback via JavaScript Event-Dispatch
-          const changed = await page.evaluate(() => {
-            const select = document.querySelector('#reviewListSorters, [data-testid="reviews-sorter-component"], select[name="reviewListSorters"]');
-            if (select) {
-              select.value = 'NEWEST_FIRST';
-              select.dispatchEvent(new Event('change', { bubbles: true }));
-              return true;
-            }
-            return false;
-          }).catch(() => false);
-
-          if (changed) {
-            console.log(`[Job ${job.id}] 🔄 Sortierung via JS-Event auf 'NEWEST_FIRST' umgestellt.`);
-            await page.waitForTimeout(2500);
-          }
         }
       } catch (e) {
         console.log(`[Job ${job.id}] ⚠️ Sortierung konnte nicht angepasst werden:`, e.message);
@@ -671,8 +660,10 @@ const worker = new Worker('scrape-jobs', async (job) => {
         pageCount++;
         console.log(`[Job ${job.id}] 📄 Scanne Bewertungsseite ${pageCount} (Bisher gesammelt: ${extractedReviews.length})...`);
 
-        const evalResult = await page.evaluate(({ westernCountries }) => {
+        const evalResult = await safeEvaluate(page, ({ westernCountries }) => {
           let cards = Array.from(document.querySelectorAll([
+            '#reviewCardsSection [data-testid="review-card"]',
+            'div[role="dialog"] [data-testid="review-card"]',
             '[data-testid="review-card"]',
             '[data-testid="review-cards"] > li',
             '[data-testid="review-cards"] > div',
@@ -681,20 +672,24 @@ const worker = new Worker('scrape-jobs', async (job) => {
             'li.review_list_new_item_block',
             'div.c-review-block__item',
             'div[itemprop="review"]',
-            'li.review_item'
+            'li.review_item',
+            'div.c-review',
+            'div.review_item',
+            '.c-review-block',
+            'li.c-review-block'
           ].join(',')));
 
           if (cards.length === 0) {
-            const container = document.querySelector('#hp-reviews-sliding, #reviewCardsSection, [data-testid="review-cards"], #blockdisplay4');
+            const container = document.querySelector('#hp-reviews-sliding, #reviewCardsSection, [data-testid="review-cards"], div[role="dialog"]');
             if (container) {
-              cards = Array.from(container.querySelectorAll('li, div')).filter(el => (el.innerText || '').length > 40);
+              cards = Array.from(container.querySelectorAll('li, div.c-review-block, div.c-review')).filter(el => (el.innerText || '').length > 30);
             }
           }
 
           // Filterboxen und Scorecard-Zusammenfassungen explizit ausschließen
           cards = cards.filter(el => {
             const text = el.innerText || '';
-            return text.length > 25 &&
+            return text.length > 20 &&
                    !text.includes('Wählen Sie Themen') && 
                    !text.includes('Kategorien:') &&
                    !el.classList.contains('review_score_breakdown');
@@ -710,15 +705,18 @@ const worker = new Worker('scrape-jobs', async (job) => {
             if (!cardText.trim()) continue;
 
             // 1. Herkunftsland des Gastes ermitteln
-            const countryEl = card.querySelector('[data-testid="review-country"], .bui-avatar-or-image__title, .user_country, .c-review__country');
+            const flagImg = card.querySelector('[data-testid="review-avatar"] .fff1944c52 img[alt], [data-testid="review-country"] img[alt], img.b8d1620349');
-            const guestCountryText = countryEl ? countryEl.innerText : cardText;
+            const countrySpan = card.querySelector('[data-testid="review-avatar"] .fff1944c52 span:not(.cec4c9cfef), [data-testid="review-country"]');
+            const guestCountryText = (flagImg ? flagImg.getAttribute('alt') : (countrySpan ? countrySpan.innerText : '')).trim();
 
+            if (guestCountryText && guestCountryText.length > 1) {
               const isWestern = westernCountries.some(c => guestCountryText.toLowerCase().includes(c.toLowerCase()));
               if (!isWestern) {
                 const preview = guestCountryText.substring(0, 50).replace(/\n/g, ' ');
                 debugLogs.push(`Karte #${idx + 1}: ❌ Land nicht in Liste ("${preview}")`);
                 continue;
               }
+            }
 
             // 2. Datum / Jahr prüfen (letzte 2 Jahre)
             const yearMatch = cardText.match(/\b(20\d{2})\b/);
@@ -737,8 +735,8 @@ const worker = new Worker('scrape-jobs', async (job) => {
             const dateEl = card.querySelector('[data-testid="review-date"], .c-review__date, .review_item_date');
 
             const title = titleEl ? titleEl.innerText.trim() : '';
-            const positive = posEl ? posEl.innerText.trim() : '';
+            const positive = posEl ? posEl.innerText.trim().replace(/^„|“$/g, '').trim() : '';
-            const negative = negEl ? negEl.innerText.trim() : '';
+            const negative = negEl ? negEl.innerText.trim().replace(/^„|“$/g, '').trim() : '';
             const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell');
 
             let fallbackText = '';
@@ -757,7 +755,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
             results,
             debugLogs: debugLogs.slice(0, 8)
           };
-        }, { westernCountries: WESTERN_COUNTRIES });
+        }, { totalCards: 0, results: [], debugLogs: [] }, { westernCountries: WESTERN_COUNTRIES });
 
         console.log(`[Job ${job.id}] 🔍 Echte Gästekarten gefunden: ${evalResult.totalCards}`);
         if (evalResult.debugLogs) {
@@ -778,7 +776,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
         // Weiterblättern im Modal oder der Hauptseite
         try {
           const nextBtn = await page.$(
-            '#hp-reviews-sliding button[aria-label*="Nächste"], #hp-reviews-sliding button[aria-label*="Next"], #reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"], a.paging-next, li.bui-pagination__next-arrow a'
+            'div[role="dialog"] button[aria-label="Nächste Seite"], button[aria-label="Nächste Seite"], button[aria-label*="Nächste Seite"], #hp-reviews-sliding button[aria-label*="Nächste"], button[aria-label*="Next"], #reviewCardsSection button:has(svg), a.paging-next, li.bui-pagination__next-arrow a'
           );
           if (nextBtn) {
             const isDisabled = await nextBtn.getAttribute('disabled');