fix: resolve AWS WAF challenge, add URL sanitization, robust DOM/JSON-LD fallback extraction, and update README

.gitignore

@@ -0,0 +1,13 @@
+node_modules/
+.env
+.env.local
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Generated debug dumps
+src/*.html
+src/*.png
+debug/
+
+.DS_Store

README.md

@@ -0,0 +1,54 @@
+# 🕵️ Scraper Service
+
+A robust background scraping service built with **Playwright**, **BullMQ**, and **Chromium** to scrape hotel metadata, real-time prices, Genius discounts, and guest reviews from Booking.com.
+
+---
+
+## 🌟 Key Capabilities
+
+- **Deep Schema & Metadata Extraction:**
+  - Unpacks nested Schema.org JSON-LD scripts (including `@graph` arrays and accommodation schemas: `Hotel`, `Resort`, `Apartment`, `Hostel`, `LodgingBusiness`, etc.).
+  - Multi-tier DOM and OpenGraph meta tag fallbacks for title, address, hero images, and ratings.
+  - Automatic URL slug parser (`extractHotelNameFromUrl`) as an ultimate fallback if client-side rendering is obstructed.
+- **AWS WAF Challenge & Anti-Bot Evasion:**
+  - **URL Sanitization (`cleanBookingUrl`):** Automatically cleans tracking tokens and expired challenge parameters (`chal_t`, `force_referer`, `sid`, `srpvid`, `srepoch`).
+  - **Browser Fingerprint Masking:** Emulates standard desktop Chrome headers, `window.chrome` runtime objects, `navigator.plugins`, and German language headers.
+  - **Active WAF Challenge Resolution:** Automatically detects AWS WAF challenge pages and waits for client-side JavaScript token generation and redirection.
+- **Overlay & Banner Handling:**
+  - Auto-dismisses OneTrust cookie banners, modal dialogs, and selection popups before evaluating DOM data.
+- **Dual-Mode Scraping:**
+  - **Quick Mode (Stammdaten & Prices):** Scrapes name, star rating, review count, address, hero image, room size, and prices in ~3–5 seconds and posts directly to Next.js webhook without LLM overhead.
+  - **Full AI Review Mode:** Sorts reviews by newest first, pages through reviews, filters for western travelers from the last 2 years, pulls Google Maps bathroom photos via SerpApi, and sends reviews to n8n for LLM summarization.
+- **Debugging & Inspection:**
+  - Automatically captures post-render rendered DOM (`src/hotel-debug.html`) and screenshots (`src/hotel-debug.png`) when `DEBUG_HTML=true` or when requested via `?debug=true`.
+
+---
+
+## ⚙️ Configuration (`.env` / Docker Environment)
+
+| Variable | Description | Default |
+|---|---|---|
+| `REDIS_HOST` | Redis host or container name | `redis` |
+| `REDIS_PORT` | Redis port | `6379` |
+| `DEBUG_HTML` | Enable saving `hotel-debug.html` and `hotel-debug.png` | `true` |
+| `SERPAPI_KEY` | (Optional) SerpApi key for Google Maps bathroom photos | `null` |
+
+---
+
+## 🚀 Running with Docker Compose
+
+```bash
+docker compose up -d --build
+```
+
+### Viewing Logs
+```bash
+docker logs -f scraper-app
+```
+
+### Inspecting Scraped DOM Debug Files
+When debugging hotel pages, the rendered DOM and screenshot are saved directly to:
+- `src/hotel-debug.html`
+- `src/hotel-debug.png`
+- `src/review-debug.png`
+

docker-compose.yml

@@ -17,6 +17,7 @@ services:
       - REDIS_HOST=redis
       - REDIS_PORT=6379
       - SERPAPI_KEY=${SERPAPI_KEY}
+      - DEBUG_HTML=${DEBUG_HTML:-true}
     env_file:
       - .env
     volumes:

src/index.js

@@ -1,7 +1,11 @@
 import { Worker } from 'bullmq';
 import { chromium } from 'playwright';
 import axios from 'axios';
+import fs from 'node:fs/promises';
 
+// ==========================================
+// KONFIGURATION & LISTEN
+// ==========================================
 const REDIS_CONFIG = {
   connection: {
     host: process.env.REDIS_HOST || 'redis',
@@ -9,40 +13,39 @@ const REDIS_CONFIG = {
   }
 };
 
+const DEBUG_HTML = process.env.DEBUG_HTML === 'true' || process.env.DEBUG_HTML === '1';
+
 const SERPAPI_KEY = process.env.SERPAPI_KEY || null;
 
 const WESTERN_COUNTRIES = [
-  'Deutschland', 'Schweiz', 'Österreich', 'Vereinigte Staaten', 'USA', 
-  'Vereinigtes Königreich', 'Großbritannien', 'Niederlande', 'Frankreich', 
-  'Spanien', 'Italien', 'Kanada', 'Australien', 'Belgien', 'Schweden', 
-  'Norwegen', 'Dänemark', 'Finnland', 'Irland', 'Luxemburg', 'Indien'
+  'Deutschland', 'Schweiz', 'Österreich', 'Germany', 'Switzerland', 'Austria',
+  'Vereinigte Staaten', 'USA', 'United States', 'Vereinigtes Königreich', 'Großbritannien',
+  'United Kingdom', 'UK', 'Niederlande', 'Netherlands', 'Frankreich', 'France',
+  'Spanien', 'Spain', 'Italien', 'Italy', 'Kanada', 'Canada', 'Australien', 'Australia',
+  'Belgien', 'Belgium', 'Schweden', 'Sweden', 'Norwegen', 'Norway', 'Dänemark', 'Denmark',
+  'Finnland', 'Finland', 'Irland', 'Ireland', 'Luxemburg', 'Luxembourg', 'Indien', 'India'
 ];
 
-// 1. Google Maps Fotos via SerpApi abrufen (inkl. Adressbereinigung & Fallback)
+// ==========================================
+// HILFSFUNKTION: GOOGLE MAPS BAD-FOTOS VIA SERPAPI
+// ==========================================
 async function getGoogleBathroomPhotosViaApi(hotelName, hotelAddress) {
   if (!SERPAPI_KEY) {
-    console.log('⚠️ [SerpApi] Kein SERPAPI_KEY in der .env hinterlegt!');
+    console.log('ℹ️ [SerpApi] Kein SERPAPI_KEY konfiguriert – überspringe Google-Maps-Fotos.');
     return [];
   }
 
   try {
-    // Booking-Adresse bereinigen: Nur Stadt und Land nutzen für maximale Trefferquote
     let cleanLocation = '';
     if (hotelAddress) {
       const parts = hotelAddress.split(',').map(p => p.trim());
       cleanLocation = parts.length > 1 ? parts.slice(-2).join(' ') : hotelAddress;
     }
     const query = `${hotelName} ${cleanLocation}`.trim();
-    console.log(`🔍 [SerpApi] Suche Google Maps Place für: "${query}"...`);
+    console.log(`🔍 [SerpApi] Suche Google Maps Ort für: "${query}"...`);
 
-    // A: Google Maps Ort suchen
     const searchRes = await axios.get('https://serpapi.com/search.json', {
-      params: {
-        engine: 'google_maps',
-        q: query,
-        hl: 'de',
-        api_key: SERPAPI_KEY
-      },
+      params: { engine: 'google_maps', q: query, hl: 'de', api_key: SERPAPI_KEY },
       timeout: 15000
     });
 
@@ -55,73 +58,99 @@ async function getGoogleBathroomPhotosViaApi(hotelName, hotelAddress) {
       return [];
     }
 
-    console.log(`📸 [SerpApi] data_id gefunden (${dataId}). Lade Fotoliste...`);
-
-    // B: Fotos & Kategorien abrufen
     const photosRes = await axios.get('https://serpapi.com/search.json', {
-      params: {
-        engine: 'google_maps_photos',
-        data_id: dataId,
-        hl: 'de',
-        api_key: SERPAPI_KEY
-      },
+      params: { engine: 'google_maps_photos', data_id: dataId, hl: 'de', api_key: SERPAPI_KEY },
       timeout: 15000
     });
 
     const categories = photosRes.data.categories || [];
-    // Flexible Erkennung für Bad-, Zimmer- und Innen-Tags
+    const strictBathKeywords = ['badezimmer', 'bad', 'bathroom', 'wc', 'toilet', 'washroom', 'restroom'];
     const bathCat = categories.find(c => {
-      const title = (c.title || '').toLowerCase();
-      return title.includes('bad') || title.includes('zimmer') || title.includes('room') || title.includes('innen') || title.includes('gast');
+      const title = (c.title || '').toLowerCase().trim();
+      return strictBathKeywords.some(kw => title.includes(kw));
     });
 
-    let photosList = photosRes.data.photos || [];
-
-    // Falls Kategorie vorhanden, gezielt Unterkategorie laden
+    let photosList = [];
     if (bathCat?.id) {
-      console.log(`🚽 [SerpApi] Kategorie gefunden: "${bathCat.title}"`);
-      try {
-        const catRes = await axios.get('https://serpapi.com/search.json', {
-          params: {
-            engine: 'google_maps_photos',
-            data_id: dataId,
-            category_id: bathCat.id,
-            hl: 'de',
-            api_key: SERPAPI_KEY
-          },
-          timeout: 15000
-        });
-        if (catRes.data.photos && catRes.data.photos.length > 0) {
-          photosList = catRes.data.photos;
-        }
-      } catch (e) {
-        console.log('⚠️ [SerpApi] Fallback auf allgemeine Fotos aktiv.');
-      }
+      console.log(`🚽 [SerpApi] Bad-Kategorie gefunden: "${bathCat.title}"`);
+      const catRes = await axios.get('https://serpapi.com/search.json', {
+        params: { engine: 'google_maps_photos', data_id: dataId, category_id: bathCat.id, hl: 'de', api_key: SERPAPI_KEY },
+        timeout: 15000
+      });
+      if (catRes.data.photos?.length > 0) photosList = catRes.data.photos;
+    } else {
+      console.log('ℹ️ [SerpApi] Kein separater Bad-Ordner auf Google Maps vorhanden.');
+      return [];
     }
 
-    // URLs bereinigen und auf HD-Auflösung setzen
-    const imageUrls = photosList
+    return photosList
       .map(p => p.image || p.thumbnail)
       .filter(Boolean)
       .map(url => url.replace(/=w\d+-h\d+[^=]*/, '=w1200-h800-k-no'))
       .slice(0, 12);
-
-    console.log(`✅ [SerpApi] ${imageUrls.length} Gästefotos erfolgreich geladen.`);
-    return imageUrls;
-
   } catch (error) {
-    console.error('❌ [SerpApi] Fehler:', error.response?.data?.error || error.message);
+    console.error('❌ [SerpApi] Fehler beim Fotoabruf:', error.message);
     return [];
   }
 }
 
-// 2. Worker Definition
-const worker = new Worker('scrape-jobs', async (job) => {
-  const { url, webhookUrl, appWebhookUrl, hotelId } = job.data;
-  const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http'));
-  console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews + Bad-Fotos)' : 'PREIS-SCAN (Schnell)'} für: ${url}`);
+// ==========================================
+// HILFSFUNKTION: HOTELNAME AUS URL EXTRAHIEREN
+// ==========================================
+function extractHotelNameFromUrl(urlString) {
+  try {
+    const parsed = new URL(urlString);
+    const parts = parsed.pathname.split('/').filter(Boolean);
+    const lastPart = parts[parts.length - 1] || '';
+    const cleanSlug = lastPart.replace(/\.[a-z]{2}\.html$/i, '').replace(/\.html$/i, '');
+    if (cleanSlug && cleanSlug.length > 2 && cleanSlug !== 'hotel') {
+      return cleanSlug
+        .split('-')
+        .map(w => w.charAt(0).toUpperCase() + w.slice(1))
+        .join(' ');
+    }
+  } catch (e) {}
+  return null;
+}
+
+// ==========================================
+// HILFSFUNKTION: URL VON TRACKING & CHALLENGE-PARAMETERN BEREINIGEN
+// ==========================================
+function cleanBookingUrl(rawUrl) {
+  try {
+    const parsed = new URL(rawUrl);
+    // Behalte nur wesentliche Buchungsparameter und entferne challenge tokens (chal_t, force_referer, etc.)
+    const allowedParams = ['checkin', 'checkout', 'group_adults', 'no_rooms', 'group_children', 'selected_currency', 'lang'];
+    const newParams = new URLSearchParams();
+
+    for (const param of allowedParams) {
+      const val = parsed.searchParams.get(param);
+      if (val) newParams.set(param, val);
+    }
+
+    if (!newParams.has('lang')) newParams.set('lang', 'de');
+    if (!newParams.has('selected_currency')) newParams.set('selected_currency', 'EUR');
+
+    parsed.search = newParams.toString();
+    parsed.hash = '';
+    return parsed.toString();
+  } catch (e) {
+    return rawUrl;
+  }
+}
+
+// ==========================================
+// BULLMQ WORKER: SCRAPING & REVIEWS
+// ==========================================
+const worker = new Worker('scrape-jobs', async (job) => {
+  const { url, webhookUrl, appWebhookUrl, hotelId, debug } = job.data;
+  const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http'));
+  const isDebugMode = Boolean(debug || DEBUG_HTML);
+  
+  console.log(`\n==================================================`);
+  console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews + Fotos)' : 'PREIS-SCAN (Schnell)'} für: ${url}`);
+  console.log(`==================================================`);
   
-  // Alle Variablen fest vorinitialisieren
   let scrapedPrice = null;
   let hasGenius = false;
   let roomSizeM2 = null;
@@ -132,22 +161,126 @@ const worker = new Worker('scrape-jobs', async (job) => {
   let browser = null;
 
   try {
-    browser = await chromium.launch({ headless: true });
-    const context = await browser.newContext({
-      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
-      viewport: { width: 1280, height: 900 },
-      locale: 'de-DE',
-      timezoneId: 'Europe/Berlin'
+    // 1. Browser mit Anti-Bot & Stealth-Optionen starten
+    browser = await chromium.launch({
+      headless: true,
+      args: [
+        '--no-sandbox',
+        '--disable-setuid-sandbox',
+        '--disable-blink-features=AutomationControlled',
+        '--disable-infobars'
+      ]
     });
+
+    const context = await browser.newContext({
+      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
+      viewport: { width: 1366, height: 900 },
+      locale: 'de-DE',
+      timezoneId: 'Europe/Berlin',
+      extraHTTPHeaders: {
+        'Accept-Language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
+        'Upgrade-Insecure-Requests': '1',
+        'Sec-Ch-Ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"',
+        'Sec-Ch-Ua-Mobile': '?0',
+        'Sec-Ch-Ua-Platform': '"Windows"'
+      }
+    });
+
     const page = await context.newPage();
 
-    let targetUrl = url;
+    // Verhindert Erkennung über navigator.webdriver und emuliert Standard-Browser-Eigenschaften
+    await page.addInitScript(() => {
+      Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+      window.chrome = {
+        runtime: {},
+        loadTimes: function() {},
+        csi: function() {},
+        app: {}
+      };
+      Object.defineProperty(navigator, 'plugins', {
+        get: () => [1, 2, 3, 4, 5]
+      });
+      Object.defineProperty(navigator, 'languages', {
+        get: () => ['de-DE', 'de', 'en-US', 'en']
+      });
+    });
+
+    let targetUrl = cleanBookingUrl(url);
     if (isReviewScan && !targetUrl.includes('#tab-reviews')) {
       targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews';
     }
 
-    await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 35000 });
-    await page.waitForTimeout(2500);
+    console.log(`[Job ${job.id}] 🌐 Bereinigte Ziel-URL: ${targetUrl}`);
+
+    await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 45000 });
+    await page.waitForTimeout(2000);
+
+    // Prüfen, ob AWS WAF Challenge aktiv ist
+    const isWafChallenge = await page.evaluate(() => {
+      return Boolean(
+        document.querySelector('#challenge-container') ||
+        document.querySelector('script[src*="challenge.js"]') ||
+        window.AwsWafIntegration ||
+        (document.title === '' && (document.body ? document.body.innerText.includes('verify that you') : false))
+      );
+    });
+
+    if (isWafChallenge) {
+      console.log(`[Job ${job.id}] 🛡️ AWS WAF Challenge erkannt. Warte auf automatische Token-Weiterleitung...`);
+      try {
+        await page.waitForFunction(() => {
+          const hasChallenge = document.querySelector('#challenge-container') || document.querySelector('script[src*="challenge.js"]');
+          const hasContent = document.querySelector('h1, h2, #hp_hotel_name, [data-testid="header-title"], script[type="application/ld+json"]');
+          return !hasChallenge || Boolean(hasContent);
+        }, { timeout: 25000 });
+        console.log(`[Job ${job.id}] 🔓 Weiterleitung nach AWS WAF erfolgt!`);
+        await page.waitForTimeout(2000);
+      } catch (wafErr) {
+        console.log(`[Job ${job.id}] ⚠️ AWS WAF Timeout. Versuche direkten Reload der Basis-URL...`);
+        try {
+          const parsed = new URL(targetUrl);
+          await page.goto(parsed.origin + parsed.pathname, { waitUntil: 'domcontentloaded', timeout: 30000 });
+          await page.waitForTimeout(3000);
+        } catch (reloadErr) {}
+      }
+    }
+
+    // Warte kurz auf wichtige Hotel-Elemente
+    try {
+      await page.waitForSelector('h1, h2, #hp_hotel_name, [data-testid="header-title"], script[type="application/ld+json"]', { timeout: 8000 });
+    } catch (e) {}
+
+    // Cookie-Banner & störende Overlays für alle Scans sofort schließen
+    try {
+      const dismissSelectors = [
+        '#onetrust-accept-btn-handler',
+        'button#onetrust-reject-all-handler',
+        'button[data-testid="selection-modal-close-btn"]',
+        'button[aria-label="Schließen"]',
+        'button[aria-label="Close"]',
+        'button.modal-mask-closeBtn'
+      ];
+      for (const btnSel of dismissSelectors) {
+        const btn = await page.$(btnSel);
+        if (btn) {
+          await btn.click().catch(() => {});
+          await page.waitForTimeout(300);
+        }
+      }
+    } catch (e) {}
+
+    // HTML-Dump & Screenshot bei Debug-Modus sichern
+    if (isDebugMode) {
+      try {
+        const htmlContent = await page.content();
+        await fs.writeFile('src/hotel-debug.html', htmlContent, 'utf-8');
+        await page.screenshot({ path: 'src/hotel-debug.png', fullPage: false });
+        console.log(`[Job ${job.id}] 📝 Debug-HTML gespeichert unter: scraper-service/src/hotel-debug.html`);
+        console.log(`[Job ${job.id}] 📸 Debug-Screenshot gespeichert unter: scraper-service/src/hotel-debug.png`);
+      } catch (err) {
+        console.error(`[Job ${job.id}] ⚠️ Fehler beim Speichern der Debug-Dateien:`, err.message);
+      }
+    }
 
     // DATUM AUS URL PARSEN
     try {
@@ -156,29 +289,175 @@ const worker = new Worker('scrape-jobs', async (job) => {
       domData.checkoutDate = parsedUrl.searchParams.get('checkout');
     } catch (e) {}
 
-    // 1. JSON-LD STAMMDATEN
+    // 2. STAMMDATEN EXTRAHIEREN (JSON-LD + DOM + META FALLBACKS)
     hotelDetails = await page.evaluate(() => {
+      let name = null;
+      let rating = null;
+      let reviewCount = null;
+      let address = null;
+      let image = null;
+
+      // A. JSON-LD durchsuchen (inkl. @graph und verschiedenen Hotel-Typen)
       try {
         const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
         for (const script of scripts) {
           try {
-            const json = JSON.parse(script.innerText || '{}');
-            if (json['@type'] === 'Hotel' || json['@type'] === 'LodgingBusiness' || json.name) {
-              return {
-                name: json.name || null,
-                rating: json.aggregateRating?.ratingValue || null,
-                reviewCount: json.aggregateRating?.reviewCount || null,
-                address: json.address?.streetAddress || json.address?.addressLocality || null,
-                image: Array.isArray(json.image) ? json.image[0] : (json.image || null)
-              };
+            const raw = JSON.parse(script.textContent || script.innerText || '{}');
+            const items = Array.isArray(raw) ? raw : (raw['@graph'] ? raw['@graph'] : [raw]);
+
+            for (const json of items) {
+              const type = (json['@type'] || '').toLowerCase();
+              const isAccommodation = ['hotel', 'lodgingbusiness', 'resort', 'apartment', 'hostel', 'motel', 'bedandbreakfast', 'guesthouse', 'inn', 'accommodation'].some(t => type.includes(t));
+
+              if (isAccommodation || (json.name && (json.address || json.aggregateRating))) {
+                if (!name && json.name && typeof json.name === 'string') {
+                  name = json.name.trim();
+                }
+                if (!rating && json.aggregateRating?.ratingValue) {
+                  const r = parseFloat(String(json.aggregateRating.ratingValue).replace(',', '.'));
+                  if (!isNaN(r)) rating = r;
+                }
+                if (!reviewCount && json.aggregateRating?.reviewCount) {
+                  const rc = parseInt(String(json.aggregateRating.reviewCount).replace(/\D/g, ''), 10);
+                  if (!isNaN(rc)) reviewCount = rc;
+                }
+                if (!address && json.address) {
+                  if (typeof json.address === 'string') {
+                    address = json.address.trim();
+                  } else if (typeof json.address === 'object') {
+                    const parts = [
+                      json.address.streetAddress,
+                      json.address.postalCode,
+                      json.address.addressLocality,
+                      json.address.addressCountry
+                    ].filter(Boolean);
+                    if (parts.length > 0) address = parts.join(', ').trim();
+                  }
+                }
+                if (!image && json.image) {
+                  if (Array.isArray(json.image) && json.image.length > 0) {
+                    image = typeof json.image[0] === 'string' ? json.image[0] : json.image[0]?.url;
+                  } else if (typeof json.image === 'string') {
+                    image = json.image;
+                  } else if (json.image?.url) {
+                    image = json.image.url;
+                  }
+                }
+              }
             }
           } catch (e) {}
         }
       } catch (e) {}
-      return { name: null, rating: null, reviewCount: null, address: null, image: null };
+
+      // B. DOM & META FALLBACKS
+
+      // 1. Name Fallback
+      if (!name) {
+        const nameSelectors = [
+          'h2.pp-header__title',
+          '[data-testid="header-title"]',
+          '#hp_hotel_name',
+          'h2.hp__hotel-name',
+          'h2.d2fee87262',
+          '.hp__hotel-title h2',
+          'h1[data-testid="header-title"]',
+          'h1'
+        ];
+        for (const sel of nameSelectors) {
+          const el = document.querySelector(sel);
+          if (el && el.innerText) {
+            const rawText = el.innerText.split('\n')[0].replace(/^(Hotel|Resort|Apartment)\s+/i, '').trim();
+            if (rawText && rawText.length > 2 && !rawText.toLowerCase().includes('booking.com')) {
+              name = rawText;
+              break;
+            }
+          }
+        }
+
+        // Meta Tag Fallback für Name
+        if (!name) {
+          const ogTitle = document.querySelector('meta[property="og:title"]')?.getAttribute('content');
+          if (ogTitle) {
+            name = ogTitle.split('–')[0].split('-')[0].split('|')[0].replace(/^(Hotel|Resort|Apartment)\s+/i, '').trim();
+          } else if (document.title) {
+            name = document.title.split('–')[0].split('-')[0].split('|')[0].replace(/^(Hotel|Resort|Apartment)\s+/i, '').trim();
+          }
+        }
+      }
+
+      // 2. Adresse Fallback
+      if (!address) {
+        const addrSelectors = [
+          '[data-testid="Address-wrapper"] span',
+          'span.hp_address_subtitle',
+          '[data-node_tt_id="location_score_tooltip"]',
+          '.bui-link--light[data-source="top_link"]',
+          'span[data-testid="address-line"]'
+        ];
+        for (const sel of addrSelectors) {
+          const el = document.querySelector(sel);
+          if (el && el.innerText && el.innerText.trim().length > 5) {
+            address = el.innerText.trim().replace(/\s+/g, ' ');
+            break;
+          }
+        }
+      }
+
+      // 3. Rating & ReviewCount Fallback
+      if (!rating || !reviewCount) {
+        try {
+          const scoreBlock = document.querySelector('[data-testid="review-score-right-component"], .b5cd09854e, .d8eab2cf7f, [data-testid="review-score-component"]');
+          const scoreText = (scoreBlock ? scoreBlock.innerText : (document.body ? document.body.innerText : '')) || '';
+
+          if (!rating && scoreText) {
+            const ratingMatch = scoreText.match(/(?:Bewertet mit|Scored|Punkte|Note|Score)?\s*([0-9][0-9]?[.,][0-9])/i);
+            if (ratingMatch) {
+              const r = parseFloat(ratingMatch[1].replace(',', '.'));
+              if (!isNaN(r) && r <= 10) rating = r;
+            }
+          }
+
+          if (!reviewCount && scoreText) {
+            const countMatch = scoreText.match(/([0-9][0-9.,]*)\s*(?:Bewertungen|reviews|rezensionen|avis|opiniones|Guest reviews)/i);
+            if (countMatch) {
+              const rc = parseInt(countMatch[1].replace(/[.,]/g, ''), 10);
+              if (!isNaN(rc)) reviewCount = rc;
+            }
+          }
+        } catch (e) {}
+      }
+
+      // 4. Hero-Image Fallback
+      if (!image) {
+        const ogImage = document.querySelector('meta[property="og:image"]')?.getAttribute('content');
+        if (ogImage && ogImage.startsWith('http')) {
+          image = ogImage;
+        } else {
+          const imgSelectors = [
+            '[data-testid="hotel-gallery"] img',
+            '.gallery_image_round_corners img',
+            '#hotel_main_content img',
+            'img.bh-photo-grid-thumb'
+          ];
+          for (const sel of imgSelectors) {
+            const imgEl = document.querySelector(sel);
+            if (imgEl && imgEl.getAttribute('src') && imgEl.getAttribute('src').startsWith('http')) {
+              image = imgEl.getAttribute('src');
+              break;
+            }
+          }
+        }
+      }
+
+      return { name, rating, reviewCount, address, image };
     });
 
-    // 2. DOM DATEN
+    // Fallback: Name aus URL-Slug ableiten wenn im HTML nichts gefunden wurde
+    if (!hotelDetails.name) {
+      hotelDetails.name = extractHotelNameFromUrl(url);
+    }
+
+    // 3. DOM-DATEN (Nächte, Personen, Zimmergröße)
     const extraDom = await page.evaluate(() => {
       try {
         const bodyText = document.body ? (document.body.innerText || '') : '';
@@ -201,74 +480,261 @@ const worker = new Worker('scrape-jobs', async (job) => {
 
     // PREIS AUSLESEN
     scrapedPrice = await page.evaluate(() => {
-      const selectors = [
-        '[data-testid="price-and-discounted-price"]',
-        '.totalPrice .bui-price-display__value',
-        '.hprt-price-price',
-        '.prco-valign-middle-helper'
-      ];
-      for (const sel of selectors) {
-        const el = document.querySelector(sel);
-        if (el && el.innerText && /\d/.test(el.innerText)) return el.innerText.trim();
-      }
+      try {
+        const selectors = [
+          '[data-testid="price-and-discounted-price"]',
+          '.totalPrice .bui-price-display__value',
+          '.hprt-price-price',
+          '.prco-valign-middle-helper',
+          '[data-testid="price-display"]',
+          'span.prco-inline-block-maker-helper',
+          '.bui-price-display__value'
+        ];
+        for (const sel of selectors) {
+          const el = document.querySelector(sel);
+          if (el && el.innerText && /\d/.test(el.innerText)) return el.innerText.trim();
+        }
+
+        const bodyText = document.body ? (document.body.innerText || '') : '';
+        const priceMatch = bodyText.match(/(?:€|EUR)\s*([0-9][0-9.,]*)|([0-9][0-9.,]*)\s*(?:€|EUR)/i);
+        if (priceMatch) {
+          return (priceMatch[1] || priceMatch[2] || '').trim();
+        }
+      } catch (e) {}
       return null;
     });
 
-    // GENIUS BADGE ERKENNUNG
+    // GENIUS BADGE PRÜFUNG
     hasGenius = await page.evaluate(() => {
-      const geniusSelectors = [
-        '.m-badge__genius',
-        'svg.genius-badge',
-        '.-genius-new_identity-genius_badge',
-        '.c-deals-container__genius-badge-fix',
-        '[aria-label*="Genius"]',
-        '[data-testid="deal-badge"]'
-      ];
-      return geniusSelectors.some(sel => Boolean(document.querySelector(sel)));
+      try {
+        const geniusSelectors = [
+          '.m-badge__genius',
+          'svg.genius-badge',
+          '.-genius-new_identity-genius_badge',
+          '.c-deals-container__genius-badge-fix',
+          '[aria-label*="Genius"]',
+          '[data-testid="deal-badge"]'
+        ];
+        return geniusSelectors.some(sel => Boolean(document.querySelector(sel)));
+      } catch (e) {
+        return false;
+      }
+    });
+
+    console.log(`[Job ${job.id}] 📋 Extrahierte Stammdaten:`, {
+      name: hotelDetails.name,
+      rating: hotelDetails.rating,
+      reviewCount: hotelDetails.reviewCount,
+      address: hotelDetails.address,
+      image: hotelDetails.image ? (hotelDetails.image.substring(0, 60) + '...') : null,
+      price: scrapedPrice,
+      hasGenius,
+      nights: domData.dNights,
+      persons: domData.dPersons,
+      roomSizeM2
     });
 
     if (hasGenius) {
       console.log(`[Job ${job.id}] 💙 Genius-Badge erkannt (-10% Rabatt gilt)!`);
     }
 
-    // 3. KOMMENTARE EXTRAHIEREN
+// 4. REVIEWS EXTRAHIEREN (STICKY-HEADER AKTIVIERUNG & ECHTER MAUSKLICK)
     if (isReviewScan) {
-      console.log(`[Job ${job.id}] Starte Review-Extraction mit Pagination...`);
+      console.log(`[Job ${job.id}] 🚀 Starte gezielte Review-Extraction für "${hotelDetails.name || 'Hotel'}"...`);
+
+      // A. Cookie-Banner und störende Overlays schließen
       try {
-        const reviewTab = await page.$('a[href*="#tab-reviews"], #js--hp-gallery-scorecard a, [data-testid="review-score-link"]');
-        if (reviewTab) {
-          await reviewTab.click();
-          await page.waitForTimeout(2000);
+        const dismissSelectors = [
+          '#onetrust-accept-btn-handler',
+          'button#onetrust-reject-all-handler',
+          'button[data-testid="selection-modal-close-btn"]',
+          'button[aria-label="Schließen"]',
+          'button[aria-label="Close"]'
+        ];
+        for (const btnSel of dismissSelectors) {
+          const btn = await page.$(btnSel);
+          if (btn) {
+            await btn.click().catch(() => {});
+            await page.waitForTimeout(400);
+          }
         }
       } catch (e) {}
 
-      await page.evaluate(() => window.scrollBy(0, 1000));
-      await page.waitForTimeout(2000);
+      // B. 500px nach unten scrollen, damit Bookings Sticky-Header (#reviews-tab-trigger) überhaupt sichtbar & aktiv wird
+      await page.evaluate(() => window.scrollBy(0, 550));
+      await page.waitForTimeout(1500);
+
+      // C. Klick per echtem Maus-Event auf den Trigger ausführen
+      let clicked = false;
+      try {
+        // Liste möglicher Klick-Ziele (Header-Tab, Scorecard-Badge oder Text "Gästebewertungen")
+        const triggerLocators = [
+          page.locator('#reviews-tab-trigger').first(),
+          page.locator('[data-testid="Property-Header-Nav-Tab-Trigger-reviews"]').first(),
+          page.locator('a:has-text("Gästebewertungen")').first(),
+          page.locator('button:has-text("Gästebewertungen")').first(),
+          page.locator('[data-testid="review-score-right-component"]').first(),
+          page.locator('[data-testid="review-score-component"]').first()
+        ];
+
+        for (const loc of triggerLocators) {
+          if (await loc.count() > 0 && await loc.isVisible()) {
+            console.log(`[Job ${job.id}] 👉 Sichtbaren Review-Trigger gefunden. Führe Klick aus...`);
+            
+            // Reale Maus-Koordinaten ermitteln & klicken
+            const box = await loc.boundingBox();
+            if (box) {
+              await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2);
+              clicked = true;
+              await page.waitForTimeout(3500);
+              break;
+            } else {
+              await loc.click({ force: true }).catch(() => {});
+              clicked = true;
+              await page.waitForTimeout(3500);
+              break;
+            }
+          }
+        }
+
+        // Fallback: JS-Klick direkt auf das HTML-Element mit Text "Gästebewertungen"
+        if (!clicked) {
+          console.log(`[Job ${job.id}] ⚠️ Fallback: Klick per DOM-Suche nach 'Gästebewertungen'...`);
+          clicked = await page.evaluate(() => {
+            const el = Array.from(document.querySelectorAll('a, button, span, div'))
+              .find(e => e.innerText && e.innerText.includes('Gästebewertungen') && (e.id === 'reviews-tab-trigger' || e.closest('#reviews-tab-trigger') || e.getAttribute('data-testid')?.includes('reviews')));
+            if (el) {
+              const clickTarget = el.closest('a') || el.closest('button') || el;
+              clickTarget.click();
+              clickTarget.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
+              return true;
+            }
+            return false;
+          }).catch(() => false);
+
+          if (clicked) await page.waitForTimeout(3500);
+        }
+
+        // Screenshot zur visuellen Kontrolle aktualisieren
+        try {
+          await page.screenshot({ path: 'src/review-debug.png', fullPage: false });
+          console.log(`[Job ${job.id}] 📸 Screenshot aktualisiert unter: scraper-service/src/review-debug.png`);
+        } catch (e) {}
+
+      } catch (e) {
+        console.log(`[Job ${job.id}] ⚠️ Fehler beim Klick auf Review-Trigger:`, e.message);
+      }
+
+      // Warten bis Review-Karten im Sliding Panel oder DOM sichtbar sind
+      try {
+        await page.waitForSelector(
+          '[data-testid="review-card"], [data-testid="review-cards"], #hp-reviews-sliding, div[data-component="hotel/reviews/review-card"], li.review_item',
+          { timeout: 8000 }
+        );
+        console.log(`[Job ${job.id}] 👁️ Review-Bereich erfolgreich gerendert.`);
+      } catch (e) {
+        console.log(`[Job ${job.id}] ⚠️ Wartezeit abgelaufen – starte Auslesen aus dem aktuellen DOM...`);
+      }
+
+      // C. Sortierung auf "Neueste zuerst" (NEWEST_FIRST) umstellen
+      try {
+        const sorterLocator = page.locator('#reviewListSorters, [data-testid="reviews-sorter-component"], select[name="reviewListSorters"]').first();
+        if (await sorterLocator.count() > 0) {
+          console.log(`[Job ${job.id}] 🔄 Stelle Sortierung auf 'NEWEST_FIRST' (Neueste zuerst) um...`);
+          await sorterLocator.selectOption('NEWEST_FIRST');
+          await page.waitForTimeout(2500); // Warten bis Booking die Liste per AJAX neu lädt
+        } else {
+          // Fallback via JavaScript Event-Dispatch
+          const changed = await page.evaluate(() => {
+            const select = document.querySelector('#reviewListSorters, [data-testid="reviews-sorter-component"], select[name="reviewListSorters"]');
+            if (select) {
+              select.value = 'NEWEST_FIRST';
+              select.dispatchEvent(new Event('change', { bubbles: true }));
+              return true;
+            }
+            return false;
+          }).catch(() => false);
+
+          if (changed) {
+            console.log(`[Job ${job.id}] 🔄 Sortierung via JS-Event auf 'NEWEST_FIRST' umgestellt.`);
+            await page.waitForTimeout(2500);
+          }
+        }
+      } catch (e) {
+        console.log(`[Job ${job.id}] ⚠️ Sortierung konnte nicht angepasst werden:`, e.message);
+      }
 
       let pageCount = 0;
       const seenKeys = new Set();
 
-      while (extractedReviews.length < 80 && pageCount < 15) {
-        const pageReviews = await page.evaluate(({ westernCountries }) => {
-          const cards = Array.from(document.querySelectorAll('[data-testid="review-card"], [role="group"][aria-label*="Bewertung"], li.review_list_new_item_block, .review_item'));
-          const results = [];
-          const currentYear = 2026;
-          for (const card of cards) {
-            const cardText = card ? (card.innerText || '') : '';
-            if (!cardText) continue;
-            const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase()));
-            if (!isWestern) continue;
+      while (pageCount < 30) {
+        pageCount++;
+        console.log(`[Job ${job.id}] 📄 Scanne Bewertungsseite ${pageCount} (Bisher gesammelt: ${extractedReviews.length})...`);
 
+        const evalResult = await page.evaluate(({ westernCountries }) => {
+          let cards = Array.from(document.querySelectorAll([
+            '[data-testid="review-card"]',
+            '[data-testid="review-cards"] > li',
+            '[data-testid="review-cards"] > div',
+            'div[data-testid*="review-card"]',
+            'li[data-testid*="review-card"]',
+            'li.review_list_new_item_block',
+            'div.c-review-block__item',
+            'div[itemprop="review"]',
+            'li.review_item'
+          ].join(',')));
+
+          if (cards.length === 0) {
+            const container = document.querySelector('#hp-reviews-sliding, #reviewCardsSection, [data-testid="review-cards"], #blockdisplay4');
+            if (container) {
+              cards = Array.from(container.querySelectorAll('li, div')).filter(el => (el.innerText || '').length > 40);
+            }
+          }
+
+          // Filterboxen und Scorecard-Zusammenfassungen explizit ausschließen
+          cards = cards.filter(el => {
+            const text = el.innerText || '';
+            return text.length > 25 &&
+                   !text.includes('Wählen Sie Themen') && 
+                   !text.includes('Kategorien:') &&
+                   !el.classList.contains('review_score_breakdown');
+          });
+
+          const results = [];
+          const debugLogs = [];
+          const currentYear = 2026;
+
+          for (let idx = 0; idx < cards.length; idx++) {
+            const card = cards[idx];
+            const cardText = card.innerText || '';
+            if (!cardText.trim()) continue;
+
+            // 1. Herkunftsland des Gastes ermitteln
+            const countryEl = card.querySelector('[data-testid="review-country"], .bui-avatar-or-image__title, .user_country, .c-review__country');
+            const guestCountryText = countryEl ? countryEl.innerText : cardText;
+
+            const isWestern = westernCountries.some(c => guestCountryText.toLowerCase().includes(c.toLowerCase()));
+            if (!isWestern) {
+              const preview = guestCountryText.substring(0, 50).replace(/\n/g, ' ');
+              debugLogs.push(`Karte #${idx + 1}: ❌ Land nicht in Liste ("${preview}")`);
+              continue;
+            }
+
+            // 2. Datum / Jahr prüfen (letzte 2 Jahre)
             const yearMatch = cardText.match(/\b(20\d{2})\b/);
             if (yearMatch) {
               const reviewYear = parseInt(yearMatch[1], 10);
-              if (currentYear - reviewYear > 2) continue;
+              if (currentYear - reviewYear > 2) {
+                debugLogs.push(`Karte #${idx + 1}: ❌ Zu alt (${reviewYear})`);
+                continue;
+              }
             }
 
-            const titleEl = card.querySelector('[data-testid="review-title"], h4');
-            const posEl = card.querySelector('[data-testid="review-positive-text"], .review_pos');
-            const negEl = card.querySelector('[data-testid="review-negative-text"], .review_neg');
-            const dateEl = card.querySelector('[data-testid="review-date"]');
+            // 3. Texte extrahieren
+            const titleEl = card.querySelector('[data-testid="review-title"], .c-review__title, h4, .review_title');
+            const posEl = card.querySelector('[data-testid="review-positive-text"], .c-review__row--positive, .review_pos');
+            const negEl = card.querySelector('[data-testid="review-negative-text"], .c-review__row--negative, .review_neg');
+            const dateEl = card.querySelector('[data-testid="review-date"], .c-review__date, .review_item_date');
 
             const title = titleEl ? titleEl.innerText.trim() : '';
             const positive = posEl ? posEl.innerText.trim() : '';
@@ -276,16 +742,30 @@ const worker = new Worker('scrape-jobs', async (job) => {
             const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell');
 
             let fallbackText = '';
-            if (!positive && !negative) fallbackText = cardText.replace(title, '').substring(0, 300).trim();
+            if (!positive && !negative) {
+              fallbackText = cardText.replace(title, '').substring(0, 300).trim();
+            }
 
             if (title || positive || negative || fallbackText) {
               results.push({ title, positive: positive || fallbackText, negative, date: dateText });
+              debugLogs.push(`Karte #${idx + 1}: ✅ Akzeptiert ("${title || (positive ? positive.substring(0, 25) : 'Review')}")`);
             }
           }
-          return results;
-        }, { westernCountries: WESTERN_COUNTRIES }).catch(() => []);
 
-        for (const rev of pageReviews) {
+          return {
+            totalCards: cards.length,
+            results,
+            debugLogs: debugLogs.slice(0, 8)
+          };
+        }, { westernCountries: WESTERN_COUNTRIES });
+
+        console.log(`[Job ${job.id}] 🔍 Echte Gästekarten gefunden: ${evalResult.totalCards}`);
+        if (evalResult.debugLogs) {
+          evalResult.debugLogs.forEach(log => console.log(`[Job ${job.id}]   └─ ${log}`));
+        }
+
+        // Deduplizierung
+        for (const rev of evalResult.results) {
           const key = `${rev.title}-${rev.positive}-${rev.date}`;
           if (!seenKeys.has(key) && extractedReviews.length < 80) {
             seenKeys.add(key);
@@ -295,26 +775,32 @@ const worker = new Worker('scrape-jobs', async (job) => {
 
         if (extractedReviews.length >= 80) break;
 
+        // Weiterblättern im Modal oder der Hauptseite
         try {
-          const nextBtn = await page.$('#reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"]');
+          const nextBtn = await page.$(
+            '#hp-reviews-sliding button[aria-label*="Nächste"], #hp-reviews-sliding button[aria-label*="Next"], #reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"], a.paging-next, li.bui-pagination__next-arrow a'
+          );
           if (nextBtn) {
             const isDisabled = await nextBtn.getAttribute('disabled');
-            if (isDisabled !== null) break;
-            await nextBtn.click();
-            await page.waitForTimeout(2000);
-            pageCount++;
+            if (isDisabled === null) {
+              await nextBtn.click({ force: true });
+              await page.waitForTimeout(2500);
+            } else break;
           } else break;
-        } catch (e) { break; }
+        } catch (e) {
+          break;
+        }
       }
-      console.log(`[Job ${job.id}] ${extractedReviews.length} Kommentare extrahiert.`);
+
+      console.log(`[Job ${job.id}] ✅ Insgesamt ${extractedReviews.length} passende Kommentare extrahiert.`);
     }
 
   } catch (error) {
-    console.error(`[Job ${job.id}] Fehler beim Scraping:`, error.message);
+    console.error(`[Job ${job.id}] ❌ Fehler beim Scraping:`, error.message);
   } finally {
     if (browser) await browser.close();
 
-// 4. Google Maps Fotos nur bei gezieltem KI-Review-Scan abrufen (spart SerpApi-Credits)
+    // 5. GOOGLE MAPS FOTOS NUR BEI KI-REVIEW SCAN ABRUFEN
     if (isReviewScan && hotelDetails.name) {
       try {
         bathroomImages = await getGoogleBathroomPhotosViaApi(hotelDetails.name, hotelDetails.address);
@@ -324,7 +810,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
       }
     }
 
-    // 5. Preishistorie und Bad-Bilder an Travel-App Webhook senden
+    // 6. PREISHISTORIE + STAMMDATEN AN NEXT.JS WEBHOOK SENDEN
     if (appWebhookUrl) {
       try {
         await axios.post(appWebhookUrl, {
@@ -342,13 +828,13 @@ const worker = new Worker('scrape-jobs', async (job) => {
           persons: domData.dPersons || 2,
           bathroomImages: Array.isArray(bathroomImages) && bathroomImages.length > 0 ? bathroomImages : undefined
         });
-        console.log(`[Job ${job.id}] ✅ Preishistorie in Postgres gespeichert!`);
+        console.log(`[Job ${job.id}] ✅ Preishistorie & Stammdaten in Postgres gespeichert!`);
       } catch (e) {
-        console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie:`, e.message);
+        console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie-Webhook:`, e.message);
       }
     }
 
-    // 6. Reviews an n8n senden
+    // 7. REVIEWS AN N8N WEBHOOK SENDEN
     if (isReviewScan && extractedReviews.length > 0) {
       try {
         await axios.post(webhookUrl, {
@@ -359,7 +845,9 @@ const worker = new Worker('scrape-jobs', async (job) => {
           timestamp: new Date().toISOString()
         });
         console.log(`[Job ${job.id}] 🚀 ${extractedReviews.length} Reviews an n8n gesendet!`);
-      } catch (e) {}
+      } catch (e) {
+        console.error(`[Job ${job.id}] ❌ Fehler beim Senden an n8n:`, e.message);
+      }
     }
   }
 }, { ...REDIS_CONFIG, concurrency: 2 });