fix(scraper): filter reviews older than 1 year instead of 2 years

.gitignore

@@ -0,0 +1,14 @@
+node_modules/
+.env
+.env.local
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Generated debug dumps
+src/*.html
+src/*.png
+debug/
+
+.DS_Store
+

AGENTS.md

@@ -0,0 +1,21 @@
+# Engineering & Scraping Guidelines
+
+## 1. Git Commit & Repository Hygiene
+- **Conventional Commits:** Write clear, concise commit messages prefixed with standard types (e.g., `feat:`, `fix:`, `refactor:`, `docs:`).
+- **Exclude Transient Artifacts:** Never stage or commit generated debug files (e.g., `src/*.html`, `src/*.png`, large logs) to version control. Always maintain `.gitignore` to keep git packs lean and prevent remote unpacker errors on Gitea/GitHub.
+
+## 2. Web Scraping & DOM Inspection Strategy
+- **Rendered DOM Dumping for Debugging:** When debugging extraction failures on dynamic or anti-bot protected pages:
+  - Capture the post-render HTML via `await page.content()` and save it to `src/hotel-debug.html`.
+  - Capture a synchronized screenshot (`src/hotel-debug.png`).
+  - Inspect the raw rendered DOM to verify if the page was served as standard SSR, an A/B test SPA variant, or intercepted by a challenge (e.g. AWS WAF).
+- **Multi-Tier Extraction Hierarchy:** Always implement fallbacks in this order:
+  1. **Deep JSON-LD Parsing:** Traverse all `<script type="application/ld+json">` tags, including `@graph` arrays and varied accommodation schemas (`Hotel`, `Resort`, `Apartment`, `Hostel`, `LodgingBusiness`, etc.).
+  2. **DOM Selectors:** Fallback to standard test IDs, classes, and element hierarchy (`data-testid`, header titles, score components).
+  3. **Meta Tags:** Fallback to OpenGraph and Twitter meta tags (`og:title`, `og:image`, `og:description`).
+  4. **URL Slug Extraction:** As an ultimate safety net, derive clean human-readable names from the URL path if the HTML body was obstructed.
+- **Anti-Bot & WAF Handling:**
+  - Sanitize input URLs by stripping transient tokens (`chal_t`, `force_referer`, session IDs).
+  - Emulate full browser properties (`window.chrome`, `navigator.plugins`, `navigator.languages`).
+  - Actively detect WAF challenge interstitial pages and wait for client-side JavaScript token computation and automatic redirection before evaluating the DOM.
+

README.md

@@ -0,0 +1,54 @@
+# 🕵️ Scraper Service
+
+A robust background scraping service built with **Playwright**, **BullMQ**, and **Chromium** to scrape hotel metadata, real-time prices, Genius discounts, and guest reviews from Booking.com.
+
+---
+
+## 🌟 Key Capabilities
+
+- **Deep Schema & Metadata Extraction:**
+  - Unpacks nested Schema.org JSON-LD scripts (including `@graph` arrays and accommodation schemas: `Hotel`, `Resort`, `Apartment`, `Hostel`, `LodgingBusiness`, etc.).
+  - Multi-tier DOM and OpenGraph meta tag fallbacks for title, address, hero images, and ratings.
+  - Automatic URL slug parser (`extractHotelNameFromUrl`) as an ultimate fallback if client-side rendering is obstructed.
+- **AWS WAF Challenge & Anti-Bot Evasion:**
+  - **URL Sanitization (`cleanBookingUrl`):** Automatically cleans tracking tokens and expired challenge parameters (`chal_t`, `force_referer`, `sid`, `srpvid`, `srepoch`).
+  - **Browser Fingerprint Masking:** Emulates standard desktop Chrome headers, `window.chrome` runtime objects, `navigator.plugins`, and German language headers.
+  - **Active WAF Challenge Resolution:** Automatically detects AWS WAF challenge pages and waits for client-side JavaScript token generation and redirection.
+- **Overlay & Banner Handling:**
+  - Auto-dismisses OneTrust cookie banners, modal dialogs, and selection popups before evaluating DOM data.
+- **Dual-Mode Scraping:**
+  - **Quick Mode (Stammdaten & Prices):** Scrapes name, star rating, review count, address, hero image, room size, and prices in ~3–5 seconds and posts directly to Next.js webhook without LLM overhead.
+  - **Full AI Review Mode:** Sorts reviews by newest first, pages through reviews, filters for western travelers from the last 2 years, pulls Google Maps bathroom photos via SerpApi, and sends reviews to n8n for LLM summarization.
+- **Debugging & Inspection:**
+  - Automatically captures post-render rendered DOM (`src/hotel-debug.html`) and screenshots (`src/hotel-debug.png`) when `DEBUG_HTML=true` or when requested via `?debug=true`.
+
+---
+
+## ⚙️ Configuration (`.env` / Docker Environment)
+
+| Variable | Description | Default |
+|---|---|---|
+| `REDIS_HOST` | Redis host or container name | `redis` |
+| `REDIS_PORT` | Redis port | `6379` |
+| `DEBUG_HTML` | Enable saving `hotel-debug.html` and `hotel-debug.png` | `true` |
+| `SERPAPI_KEY` | (Optional) SerpApi key for Google Maps bathroom photos | `null` |
+
+---
+
+## 🚀 Running with Docker Compose
+
+```bash
+docker compose up -d --build
+```
+
+### Viewing Logs
+```bash
+docker logs -f scraper-app
+```
+
+### Inspecting Scraped DOM Debug Files
+When debugging hotel pages, the rendered DOM and screenshot are saved directly to:
+- `src/hotel-debug.html`
+- `src/hotel-debug.png`
+- `src/review-debug.png`
+

docker-compose.yml

@@ -17,6 +17,7 @@ services:
       - REDIS_HOST=redis
       - REDIS_PORT=6379
       - SERPAPI_KEY=${SERPAPI_KEY}
+      - DEBUG_HTML=${DEBUG_HTML:-true}
     env_file:
       - .env
     volumes:

src/index.js

@@ -1,7 +1,11 @@
 import { Worker } from 'bullmq';
 import { chromium } from 'playwright';
 import axios from 'axios';
+import fs from 'node:fs/promises';
 
+// ==========================================
+// KONFIGURATION & LISTEN
+// ==========================================
 const REDIS_CONFIG = {
   connection: {
     host: process.env.REDIS_HOST || 'redis',
@@ -9,40 +13,40 @@ const REDIS_CONFIG = {
   }
 };
 
+const DEBUG_HTML = process.env.DEBUG_HTML === 'true' || process.env.DEBUG_HTML === '1';
+
 const SERPAPI_KEY = process.env.SERPAPI_KEY || null;
 
 const WESTERN_COUNTRIES = [
-  'Deutschland', 'Schweiz', 'Österreich', 'Vereinigte Staaten', 'USA', 
+  'Deutschland', 'Schweiz', 'Österreich', 'Germany', 'Switzerland', 'Austria',
-  'Vereinigtes Königreich', 'Großbritannien', 'Niederlande', 'Frankreich', 
+  'Vereinigte Staaten', 'USA', 'United States', 'Vereinigtes Königreich', 'Großbritannien',
-  'Spanien', 'Italien', 'Kanada', 'Australien', 'Belgien', 'Schweden', 
+  'United Kingdom', 'UK', 'Niederlande', 'Netherlands', 'Frankreich', 'France',
-  'Norwegen', 'Dänemark', 'Finnland', 'Irland', 'Luxemburg', 'Indien'
+  'Spanien', 'Spain', 'Italien', 'Italy', 'Kanada', 'Canada', 'Australien', 'Australia',
+  'Belgien', 'Belgium', 'Schweden', 'Sweden', 'Norwegen', 'Norway', 'Dänemark', 'Denmark',
+  'Finnland', 'Finland', 'Irland', 'Ireland', 'Luxemburg', 'Luxembourg',
+  'Neuseeland', 'New Zealand'
 ];
 
-// 1. Google Maps Fotos via SerpApi abrufen (inkl. Adressbereinigung & Fallback)
+// ==========================================
+// HILFSFUNKTION: GOOGLE MAPS BAD-FOTOS VIA SERPAPI
+// ==========================================
 async function getGoogleBathroomPhotosViaApi(hotelName, hotelAddress) {
   if (!SERPAPI_KEY) {
-    console.log('⚠️ [SerpApi] Kein SERPAPI_KEY in der .env hinterlegt!');
+    console.log('ℹ️ [SerpApi] Kein SERPAPI_KEY konfiguriert – überspringe Google-Maps-Fotos.');
     return [];
   }
 
   try {
-    // Booking-Adresse bereinigen: Nur Stadt und Land nutzen für maximale Trefferquote
     let cleanLocation = '';
     if (hotelAddress) {
       const parts = hotelAddress.split(',').map(p => p.trim());
       cleanLocation = parts.length > 1 ? parts.slice(-2).join(' ') : hotelAddress;
     }
     const query = `${hotelName} ${cleanLocation}`.trim();
-    console.log(`🔍 [SerpApi] Suche Google Maps Place für: "${query}"...`);
+    console.log(`🔍 [SerpApi] Suche Google Maps Ort für: "${query}"...`);
 
-    // A: Google Maps Ort suchen
     const searchRes = await axios.get('https://serpapi.com/search.json', {
-      params: {
+      params: { engine: 'google_maps', q: query, hl: 'de', api_key: SERPAPI_KEY },
-        engine: 'google_maps',
-        q: query,
-        hl: 'de',
-        api_key: SERPAPI_KEY
-      },
       timeout: 15000
     });
 
@@ -55,73 +59,119 @@ async function getGoogleBathroomPhotosViaApi(hotelName, hotelAddress) {
       return [];
     }
 
-    console.log(`📸 [SerpApi] data_id gefunden (${dataId}). Lade Fotoliste...`);
-
-    // B: Fotos & Kategorien abrufen
     const photosRes = await axios.get('https://serpapi.com/search.json', {
-      params: {
+      params: { engine: 'google_maps_photos', data_id: dataId, hl: 'de', api_key: SERPAPI_KEY },
-        engine: 'google_maps_photos',
-        data_id: dataId,
-        hl: 'de',
-        api_key: SERPAPI_KEY
-      },
       timeout: 15000
     });
 
     const categories = photosRes.data.categories || [];
-    // Flexible Erkennung für Bad-, Zimmer- und Innen-Tags
+    const strictBathKeywords = ['badezimmer', 'bad', 'bathroom', 'wc', 'toilet', 'washroom', 'restroom'];
     const bathCat = categories.find(c => {
-      const title = (c.title || '').toLowerCase();
+      const title = (c.title || '').toLowerCase().trim();
-      return title.includes('bad') || title.includes('zimmer') || title.includes('room') || title.includes('innen') || title.includes('gast');
+      return strictBathKeywords.some(kw => title.includes(kw));
     });
 
-    let photosList = photosRes.data.photos || [];
+    let photosList = [];
-
-    // Falls Kategorie vorhanden, gezielt Unterkategorie laden
     if (bathCat?.id) {
-      console.log(`🚽 [SerpApi] Kategorie gefunden: "${bathCat.title}"`);
+      console.log(`🚽 [SerpApi] Bad-Kategorie gefunden: "${bathCat.title}"`);
-      try {
       const catRes = await axios.get('https://serpapi.com/search.json', {
-          params: {
+        params: { engine: 'google_maps_photos', data_id: dataId, category_id: bathCat.id, hl: 'de', api_key: SERPAPI_KEY },
-            engine: 'google_maps_photos',
-            data_id: dataId,
-            category_id: bathCat.id,
-            hl: 'de',
-            api_key: SERPAPI_KEY
-          },
         timeout: 15000
       });
-        if (catRes.data.photos && catRes.data.photos.length > 0) {
+      if (catRes.data.photos?.length > 0) photosList = catRes.data.photos;
-          photosList = catRes.data.photos;
+    } else {
-        }
+      console.log('ℹ️ [SerpApi] Kein separater Bad-Ordner auf Google Maps vorhanden.');
-      } catch (e) {
+      return [];
-        console.log('⚠️ [SerpApi] Fallback auf allgemeine Fotos aktiv.');
-      }
     }
 
-    // URLs bereinigen und auf HD-Auflösung setzen
+    return photosList
-    const imageUrls = photosList
       .map(p => p.image || p.thumbnail)
       .filter(Boolean)
       .map(url => url.replace(/=w\d+-h\d+[^=]*/, '=w1200-h800-k-no'))
       .slice(0, 12);
-
-    console.log(`✅ [SerpApi] ${imageUrls.length} Gästefotos erfolgreich geladen.`);
-    return imageUrls;
-
   } catch (error) {
-    console.error('❌ [SerpApi] Fehler:', error.response?.data?.error || error.message);
+    console.error('❌ [SerpApi] Fehler beim Fotoabruf:', error.message);
     return [];
   }
 }
 
-// 2. Worker Definition
+// ==========================================
-const worker = new Worker('scrape-jobs', async (job) => {
+// HILFSFUNKTION: HOTELNAME AUS URL EXTRAHIEREN
-  const { url, webhookUrl, appWebhookUrl, hotelId } = job.data;
+// ==========================================
-  const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http'));
+function extractHotelNameFromUrl(urlString) {
-  console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews + Bad-Fotos)' : 'PREIS-SCAN (Schnell)'} für: ${url}`);
+  try {
+    const parsed = new URL(urlString);
+    const parts = parsed.pathname.split('/').filter(Boolean);
+    const lastPart = parts[parts.length - 1] || '';
+    const cleanSlug = lastPart.replace(/\.[a-z]{2}\.html$/i, '').replace(/\.html$/i, '');
+    if (cleanSlug && cleanSlug.length > 2 && cleanSlug !== 'hotel') {
+      return cleanSlug
+        .split('-')
+        .map(w => w.charAt(0).toUpperCase() + w.slice(1))
+        .join(' ');
+    }
+  } catch (e) {}
+  return null;
+}
+
+// ==========================================
+// HILFSFUNKTION: URL VON TRACKING & CHALLENGE-PARAMETERN BEREINIGEN
+// ==========================================
+function cleanBookingUrl(rawUrl) {
+  try {
+    const parsed = new URL(rawUrl);
+    // Behalte nur wesentliche Buchungsparameter und entferne challenge tokens (chal_t, force_referer, etc.)
+    const allowedParams = ['checkin', 'checkout', 'group_adults', 'no_rooms', 'group_children', 'selected_currency', 'lang'];
+    const newParams = new URLSearchParams();
+
+    for (const param of allowedParams) {
+      const val = parsed.searchParams.get(param);
+      if (val) newParams.set(param, val);
+    }
+
+    if (!newParams.has('lang')) newParams.set('lang', 'de');
+    if (!newParams.has('selected_currency')) newParams.set('selected_currency', 'EUR');
+
+    parsed.search = newParams.toString();
+    parsed.hash = '';
+    return parsed.toString();
+  } catch (e) {
+    return rawUrl;
+  }
+}
+
+// ==========================================
+// HILFSFUNKTION: SICHERE EVALUATION (RESISTENT GEGEN REDIRECTS)
+// ==========================================
+async function safeEvaluate(page, fn, fallback = null, arg = undefined) {
+  try {
+    return arg !== undefined ? await page.evaluate(fn, arg) : await page.evaluate(fn);
+  } catch (err) {
+    if (err.message && (err.message.includes('Execution context was destroyed') || err.message.includes('navigating') || err.message.includes('Target closed'))) {
+      await page.waitForLoadState('domcontentloaded').catch(() => {});
+      await page.waitForTimeout(2500);
+      try {
+        return arg !== undefined ? await page.evaluate(fn, arg) : await page.evaluate(fn);
+      } catch (innerErr) {
+        return fallback;
+      }
+    }
+    return fallback;
+  }
+}
+
+// ==========================================
+// BULLMQ WORKER: SCRAPING & REVIEWS
+// ==========================================
+const worker = new Worker('scrape-jobs', async (job) => {
+  const { url, webhookUrl, appWebhookUrl, hotelId, debug, mode } = job.data;
+  const isReviewScan = Boolean((webhookUrl && webhookUrl.startsWith('http')) || mode === 'reviews' || mode === 'full');
+  const isDebugMode = Boolean(debug || DEBUG_HTML);
+  
+  console.log(`\n==================================================`);
+  console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews + Fotos)' : 'PREIS-SCAN (Schnell)'} für: ${url}`);
+  console.log(`==================================================`);
   
-  // Alle Variablen fest vorinitialisieren
   let scrapedPrice = null;
   let hasGenius = false;
   let roomSizeM2 = null;
@@ -132,22 +182,129 @@ const worker = new Worker('scrape-jobs', async (job) => {
   let browser = null;
 
   try {
-    browser = await chromium.launch({ headless: true });
+    // 1. Browser mit Anti-Bot & Stealth-Optionen starten
+    browser = await chromium.launch({
+      headless: true,
+      args: [
+        '--no-sandbox',
+        '--disable-setuid-sandbox',
+        '--disable-dev-shm-usage',
+        '--disable-blink-features=AutomationControlled',
+        '--window-size=1920,1080'
+      ]
+    });
+
     const context = await browser.newContext({
-      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
-      viewport: { width: 1280, height: 900 },
+      viewport: { width: 1920, height: 1080 },
       locale: 'de-DE',
       timezoneId: 'Europe/Berlin'
     });
+
+    // Verhindert Erkennung über navigator.webdriver und emuliert Standard-Browser-Eigenschaften
+    await context.addInitScript(() => {
+      Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+      window.chrome = {
+        runtime: {},
+        loadTimes: function() {},
+        csi: function() {},
+        app: {}
+      };
+      Object.defineProperty(navigator, 'plugins', {
+        get: () => [1, 2, 3, 4, 5]
+      });
+      Object.defineProperty(navigator, 'languages', {
+        get: () => ['de-DE', 'de', 'en-US', 'en']
+      });
+    });
+
     const page = await context.newPage();
 
-    let targetUrl = url;
+    let targetUrl = cleanBookingUrl(url);
-    if (isReviewScan && !targetUrl.includes('#tab-reviews')) {
+
-      targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews';
+    console.log(`[Job ${job.id}] 🌐 Bereinigte Ziel-URL: ${targetUrl}`);
+
+    await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 45000 });
+    await page.waitForLoadState('domcontentloaded').catch(() => {});
+    await page.waitForTimeout(2000);
+
+    // Prüfen, ob AWS WAF Challenge aktiv ist (sicher gegenüber gleichzeitiger Navigation)
+    let isWafChallenge = false;
+    try {
+      isWafChallenge = await page.evaluate(() => {
+        return Boolean(
+          document.querySelector('#challenge-container') ||
+          document.querySelector('script[src*="challenge.js"]') ||
+          window.AwsWafIntegration ||
+          (document.title === '' && (document.body ? document.body.innerText.includes('verify that you') : false))
+        );
+      });
+    } catch (navErr) {
+      await page.waitForLoadState('domcontentloaded').catch(() => {});
+      await page.waitForTimeout(2500);
+      try {
+        isWafChallenge = await page.evaluate(() => {
+          return Boolean(
+            document.querySelector('#challenge-container') ||
+            document.querySelector('script[src*="challenge.js"]') ||
+            window.AwsWafIntegration
+          );
+        });
+      } catch (e) {
+        isWafChallenge = false;
+      }
     }
 
-    await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 35000 });
+    if (isWafChallenge) {
-    await page.waitForTimeout(2500);
+      console.log(`[Job ${job.id}] 🛡️ AWS WAF Challenge erkannt. Warte auf automatische Token-Weiterleitung...`);
+      try {
+        await page.waitForFunction(() => {
+          const hasChallenge = document.querySelector('#challenge-container') || document.querySelector('script[src*="challenge.js"]');
+          const hasContent = document.querySelector('h1, h2, #hp_hotel_name, [data-testid="header-title"], script[type="application/ld+json"]');
+          return !hasChallenge || Boolean(hasContent);
+        }, { timeout: 25000 });
+        console.log(`[Job ${job.id}] 🔓 Weiterleitung nach AWS WAF erfolgt!`);
+        await page.waitForLoadState('domcontentloaded').catch(() => {});
+        await page.waitForTimeout(2000);
+      } catch (wafErr) {
+        console.log(`[Job ${job.id}] ⚠️ AWS WAF Timeout. Versuche direkten Reload der Basis-URL...`);
+        try {
+          const parsed = new URL(targetUrl);
+          await page.goto(parsed.origin + parsed.pathname, { waitUntil: 'domcontentloaded', timeout: 30000 });
+          await page.waitForLoadState('domcontentloaded').catch(() => {});
+          await page.waitForTimeout(3000);
+        } catch (reloadErr) {}
+      }
+    }
+
+    // Warte kurz auf wichtige Hotel-Elemente
+    try {
+      await page.waitForSelector('h1, h2, #hp_hotel_name, [data-testid="header-title"], script[type="application/ld+json"]', { timeout: 8000 });
+    } catch (e) {}
+
+    // Cookie-Banner & störende Overlays für alle Scans sofort schließen
+    try {
+      const dismissBtns = await page.$$(
+        'button[aria-label*="schließen"], button[aria-label*="Schließen"], button[aria-label*="Close"], button[aria-label*="Dismiss"], #onetrust-accept-btn-handler, button:has-text("Akzeptieren"), button:has-text("Alle akzeptieren"), button[data-testid="selection-modal-close-btn"]'
+      );
+      for (const btn of dismissBtns) {
+        await btn.click().catch(() => {});
+        await page.waitForTimeout(300);
+      }
+    } catch (e) {}
+
+    // HTML-Dump & Screenshot bei Debug-Modus sichern
+    if (isDebugMode) {
+      try {
+        const htmlContent = await page.content();
+        await fs.writeFile('src/hotel-debug.html', htmlContent, 'utf-8');
+        await page.screenshot({ path: 'src/hotel-debug.png', fullPage: false });
+        console.log(`[Job ${job.id}] 📝 Debug-HTML gespeichert unter: scraper-service/src/hotel-debug.html`);
+        console.log(`[Job ${job.id}] 📸 Debug-Screenshot gespeichert unter: scraper-service/src/hotel-debug.png`);
+      } catch (err) {
+        console.error(`[Job ${job.id}] ⚠️ Fehler beim Speichern der Debug-Dateien:`, err.message);
+      }
+    }
 
     // DATUM AUS URL PARSEN
     try {
@@ -156,30 +313,176 @@ const worker = new Worker('scrape-jobs', async (job) => {
       domData.checkoutDate = parsedUrl.searchParams.get('checkout');
     } catch (e) {}
 
-    // 1. JSON-LD STAMMDATEN
+    // 2. STAMMDATEN EXTRAHIEREN (JSON-LD + DOM + META FALLBACKS)
-    hotelDetails = await page.evaluate(() => {
+    hotelDetails = await safeEvaluate(page, () => {
+      let name = null;
+      let rating = null;
+      let reviewCount = null;
+      let address = null;
+      let image = null;
+
+      // A. JSON-LD durchsuchen (inkl. @graph und verschiedenen Hotel-Typen)
       try {
         const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
         for (const script of scripts) {
           try {
-            const json = JSON.parse(script.innerText || '{}');
+            const raw = JSON.parse(script.textContent || script.innerText || '{}');
-            if (json['@type'] === 'Hotel' || json['@type'] === 'LodgingBusiness' || json.name) {
+            const items = Array.isArray(raw) ? raw : (raw['@graph'] ? raw['@graph'] : [raw]);
-              return {
-                name: json.name || null,
-                rating: json.aggregateRating?.ratingValue || null,
-                reviewCount: json.aggregateRating?.reviewCount || null,
-                address: json.address?.streetAddress || json.address?.addressLocality || null,
-                image: Array.isArray(json.image) ? json.image[0] : (json.image || null)
-              };
-            }
-          } catch (e) {}
-        }
-      } catch (e) {}
-      return { name: null, rating: null, reviewCount: null, address: null, image: null };
-    });
 
-    // 2. DOM DATEN
+            for (const json of items) {
-    const extraDom = await page.evaluate(() => {
+              const type = (json['@type'] || '').toLowerCase();
+              const isAccommodation = ['hotel', 'lodgingbusiness', 'resort', 'apartment', 'hostel', 'motel', 'bedandbreakfast', 'guesthouse', 'inn', 'accommodation'].some(t => type.includes(t));
+
+              if (isAccommodation || (json.name && (json.address || json.aggregateRating))) {
+                if (!name && json.name && typeof json.name === 'string') {
+                  name = json.name.trim();
+                }
+                if (!rating && json.aggregateRating?.ratingValue) {
+                  const r = parseFloat(String(json.aggregateRating.ratingValue).replace(',', '.'));
+                  if (!isNaN(r)) rating = r;
+                }
+                if (!reviewCount && json.aggregateRating?.reviewCount) {
+                  const rc = parseInt(String(json.aggregateRating.reviewCount).replace(/\D/g, ''), 10);
+                  if (!isNaN(rc)) reviewCount = rc;
+                }
+                if (!address && json.address) {
+                  if (typeof json.address === 'string') {
+                    address = json.address.trim();
+                  } else if (typeof json.address === 'object') {
+                    const parts = [
+                      json.address.streetAddress,
+                      json.address.postalCode,
+                      json.address.addressLocality,
+                      json.address.addressCountry
+                    ].filter(Boolean);
+                    if (parts.length > 0) address = parts.join(', ').trim();
+                  }
+                }
+                if (!image && json.image) {
+                  if (Array.isArray(json.image) && json.image.length > 0) {
+                    image = typeof json.image[0] === 'string' ? json.image[0] : json.image[0]?.url;
+                  } else if (typeof json.image === 'string') {
+                    image = json.image;
+                  } else if (json.image?.url) {
+                    image = json.image.url;
+                  }
+                }
+              }
+            }
+          } catch (e) {}
+        }
+      } catch (e) {}
+
+      // B. DOM & META FALLBACKS
+
+      // 1. Name Fallback
+      if (!name) {
+        const nameSelectors = [
+          'h2.pp-header__title',
+          '[data-testid="header-title"]',
+          '#hp_hotel_name',
+          'h2.hp__hotel-name',
+          'h2.d2fee87262',
+          '.hp__hotel-title h2',
+          'h1[data-testid="header-title"]',
+          'h1'
+        ];
+        for (const sel of nameSelectors) {
+          const el = document.querySelector(sel);
+          if (el && el.innerText) {
+            const rawText = el.innerText.split('\n')[0].replace(/^(Hotel|Resort|Apartment)\s+/i, '').trim();
+            if (rawText && rawText.length > 2 && !rawText.toLowerCase().includes('booking.com')) {
+              name = rawText;
+              break;
+            }
+          }
+        }
+
+        // Meta Tag Fallback für Name
+        if (!name) {
+          const ogTitle = document.querySelector('meta[property="og:title"]')?.getAttribute('content');
+          if (ogTitle) {
+            name = ogTitle.split('–')[0].split('-')[0].split('|')[0].replace(/^(Hotel|Resort|Apartment)\s+/i, '').trim();
+          } else if (document.title) {
+            name = document.title.split('–')[0].split('-')[0].split('|')[0].replace(/^(Hotel|Resort|Apartment)\s+/i, '').trim();
+          }
+        }
+      }
+
+      // 2. Adresse Fallback
+      if (!address) {
+        const addrSelectors = [
+          '[data-testid="Address-wrapper"] span',
+          'span.hp_address_subtitle',
+          '[data-node_tt_id="location_score_tooltip"]',
+          '.bui-link--light[data-source="top_link"]',
+          'span[data-testid="address-line"]'
+        ];
+        for (const sel of addrSelectors) {
+          const el = document.querySelector(sel);
+          if (el && el.innerText && el.innerText.trim().length > 5) {
+            address = el.innerText.trim().replace(/\s+/g, ' ');
+            break;
+          }
+        }
+      }
+
+      // 3. Rating & ReviewCount Fallback
+      if (!rating || !reviewCount) {
+        try {
+          const scoreBlock = document.querySelector('[data-testid="review-score-right-component"], .b5cd09854e, .d8eab2cf7f, [data-testid="review-score-component"]');
+          const scoreText = (scoreBlock ? scoreBlock.innerText : (document.body ? document.body.innerText : '')) || '';
+
+          if (!rating && scoreText) {
+            const ratingMatch = scoreText.match(/(?:Bewertet mit|Scored|Punkte|Note|Score)?\s*([0-9][0-9]?[.,][0-9])/i);
+            if (ratingMatch) {
+              const r = parseFloat(ratingMatch[1].replace(',', '.'));
+              if (!isNaN(r) && r <= 10) rating = r;
+            }
+          }
+
+          if (!reviewCount && scoreText) {
+            const countMatch = scoreText.match(/([0-9][0-9.,]*)\s*(?:Bewertungen|reviews|rezensionen|avis|opiniones|Guest reviews)/i);
+            if (countMatch) {
+              const rc = parseInt(countMatch[1].replace(/[.,]/g, ''), 10);
+              if (!isNaN(rc)) reviewCount = rc;
+            }
+          }
+        } catch (e) {}
+      }
+
+      // 4. Hero-Image Fallback
+      if (!image) {
+        const ogImage = document.querySelector('meta[property="og:image"]')?.getAttribute('content');
+        if (ogImage && ogImage.startsWith('http')) {
+          image = ogImage;
+        } else {
+          const imgSelectors = [
+            '[data-testid="hotel-gallery"] img',
+            '.gallery_image_round_corners img',
+            '#hotel_main_content img',
+            'img.bh-photo-grid-thumb'
+          ];
+          for (const sel of imgSelectors) {
+            const imgEl = document.querySelector(sel);
+            if (imgEl && imgEl.getAttribute('src') && imgEl.getAttribute('src').startsWith('http')) {
+              image = imgEl.getAttribute('src');
+              break;
+            }
+          }
+        }
+      }
+
+      return { name, rating, reviewCount, address, image };
+    }, { name: null, rating: null, reviewCount: null, address: null, image: null });
+
+    // Fallback: Name aus URL-Slug ableiten wenn im HTML nichts gefunden wurde
+    if (!hotelDetails.name) {
+      hotelDetails.name = extractHotelNameFromUrl(url);
+    }
+
+    // 3. DOM-DATEN (Nächte, Personen, Zimmergröße)
+    const extraDom = await safeEvaluate(page, () => {
       try {
         const bodyText = document.body ? (document.body.innerText || '') : '';
         const nMatch = bodyText.match(/(\d+)\s*Nächte?/i);
@@ -193,29 +496,41 @@ const worker = new Worker('scrape-jobs', async (job) => {
       } catch (e) {
         return { dNights: 1, dPersons: 2, roomSize: null };
       }
-    });
+    }, { dNights: 1, dPersons: 2, roomSize: null });
 
     domData.dNights = extraDom.dNights;
     domData.dPersons = extraDom.dPersons;
     roomSizeM2 = extraDom.roomSize;
 
     // PREIS AUSLESEN
-    scrapedPrice = await page.evaluate(() => {
+    scrapedPrice = await safeEvaluate(page, () => {
+      try {
         const selectors = [
           '[data-testid="price-and-discounted-price"]',
           '.totalPrice .bui-price-display__value',
           '.hprt-price-price',
-        '.prco-valign-middle-helper'
+          '.prco-valign-middle-helper',
+          '[data-testid="price-display"]',
+          'span.prco-inline-block-maker-helper',
+          '.bui-price-display__value'
         ];
         for (const sel of selectors) {
           const el = document.querySelector(sel);
           if (el && el.innerText && /\d/.test(el.innerText)) return el.innerText.trim();
         }
-      return null;
-    });
 
-    // GENIUS BADGE ERKENNUNG
+        const bodyText = document.body ? (document.body.innerText || '') : '';
-    hasGenius = await page.evaluate(() => {
+        const priceMatch = bodyText.match(/(?:€|EUR)\s*([0-9][0-9.,]*)|([0-9][0-9.,]*)\s*(?:€|EUR)/i);
+        if (priceMatch) {
+          return (priceMatch[1] || priceMatch[2] || '').trim();
+        }
+      } catch (e) {}
+      return null;
+    }, null);
+
+    // GENIUS BADGE PRÜFUNG
+    hasGenius = await safeEvaluate(page, () => {
+      try {
         const geniusSelectors = [
           '.m-badge__genius',
           'svg.genius-badge',
@@ -225,96 +540,175 @@ const worker = new Worker('scrape-jobs', async (job) => {
           '[data-testid="deal-badge"]'
         ];
         return geniusSelectors.some(sel => Boolean(document.querySelector(sel)));
+      } catch (e) {
+        return false;
+      }
+    }, false);
+
+    console.log(`[Job ${job.id}] 📋 Extrahierte Stammdaten:`, {
+      name: hotelDetails.name,
+      rating: hotelDetails.rating,
+      reviewCount: hotelDetails.reviewCount,
+      address: hotelDetails.address,
+      image: hotelDetails.image ? (hotelDetails.image.substring(0, 60) + '...') : null,
+      price: scrapedPrice,
+      hasGenius,
+      nights: domData.dNights,
+      persons: domData.dPersons,
+      roomSizeM2
     });
 
     if (hasGenius) {
       console.log(`[Job ${job.id}] 💙 Genius-Badge erkannt (-10% Rabatt gilt)!`);
     }
 
-    // 3. KOMMENTARE EXTRAHIEREN
+// 4. REVIEWS EXTRAHIEREN (STICKY-HEADER AKTIVIERUNG & ECHTER MAUSKLICK)
     if (isReviewScan) {
-      console.log(`[Job ${job.id}] Starte Review-Extraction mit Pagination...`);
+      console.log(`[Job ${job.id}] 🚀 Starte gezielte Review-Extraction für "${hotelDetails.name || 'Hotel'}"...`);
-      try {
+
-        const reviewTab = await page.$('a[href*="#tab-reviews"], #js--hp-gallery-scorecard a, [data-testid="review-score-link"]');
+      // B. Herunterscrollen um Lazy-Components zu hydrieren und Review-Dialog zu öffnen
-        if (reviewTab) {
+      console.log(`[Job ${job.id}] 📜 Scrolle herunter, um verzögert geladene Review-Komponenten zu aktivieren...`);
-          await reviewTab.click();
+      await page.evaluate(() => window.scrollTo(0, 4000));
-          await page.waitForTimeout(2000);
+      await page.waitForTimeout(2500);
+
+      // C. Klick auf Review-Trigger ausführen
+      console.log(`[Job ${job.id}] 👉 Öffne Review-Dialog über 'Alle Bewertungen lesen' / 'Gästebewertungen'...`);
+      await page.evaluate(() => {
+        const btn = document.querySelector('button[data-testid="fr-read-all-reviews"]') ||
+                    document.querySelector('button[data-testid="review-score-read-all-actionable"]') ||
+                    document.getElementById('reviews-tab-trigger') ||
+                    document.querySelector('[data-testid="Property-Header-Nav-Tab-Trigger-reviews"]');
+        if (btn) {
+          btn.scrollIntoView({ behavior: 'instant', block: 'center' });
+          btn.click();
+          btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
+          return true;
         }
+        return false;
+      }).catch(() => false);
+
+      // 4 Sekunden warten bis das Dialog-Panel mit den 1.000+ Bewertungen vollständig geöffnet ist
+      await page.waitForTimeout(4000);
+
+      // Screenshot zur visuellen Kontrolle aktualisieren
+      try {
+        await page.screenshot({ path: 'src/review-debug.png', fullPage: false });
+        console.log(`[Job ${job.id}] 📸 Screenshot aktualisiert unter: scraper-service/src/review-debug.png`);
       } catch (e) {}
 
-      await page.evaluate(() => window.scrollBy(0, 1000));
+      // D. Sortierung auf "Neueste zuerst" (NEWEST_FIRST) umstellen
-      await page.waitForTimeout(2000);
+      try {
+        const sorterLocator = page.locator('#reviewListSorters, [data-testid="reviews-sorter-component"], select[name="reviewListSorters"]').first();
+        if (await sorterLocator.count() > 0) {
+          console.log(`[Job ${job.id}] 🔄 Stelle Sortierung auf 'NEWEST_FIRST' (Neueste zuerst) um...`);
+          await sorterLocator.selectOption('NEWEST_FIRST');
+          await page.waitForTimeout(3000); // Warten bis Booking die Liste per AJAX neu lädt
+        }
+      } catch (e) {
+        console.log(`[Job ${job.id}] ⚠️ Sortierung konnte nicht angepasst werden:`, e.message);
+      }
 
       let pageCount = 0;
       const seenKeys = new Set();
 
-      while (extractedReviews.length < 80 && pageCount < 15) {
+      while (pageCount < 30) {
-        const pageReviews = await page.evaluate(({ westernCountries }) => {
+        pageCount++;
-          const cards = Array.from(document.querySelectorAll('[data-testid="review-card"], [role="group"][aria-label*="Bewertung"], li.review_list_new_item_block, .review_item'));
+        console.log(`[Job ${job.id}] 📄 Scanne Bewertungsseite ${pageCount} (Bisher gesammelt: ${extractedReviews.length})...`);
-          const results = [];
-          const currentYear = 2026;
-          for (const card of cards) {
-            const cardText = card ? (card.innerText || '') : '';
-            if (!cardText) continue;
-            const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase()));
-            if (!isWestern) continue;
 
+        // Extrahiere alle Review-Karten aus dem Dialog / DOM
+        const pageReviews = await page.evaluate(() => {
+          const cards = Array.from(document.querySelectorAll(
+            '#reviewCardsSection [data-testid="review-card"], div[role="dialog"] [data-testid="review-card"], [data-testid="review-card"]'
+          ));
+          const items = [];
+          const currentYear = new Date().getFullYear();
+
+          for (const card of cards) {
+            const titleEl = card.querySelector('[data-testid="review-title"], h4');
+            const posEl = card.querySelector('[data-testid="review-positive-text"]');
+            const negEl = card.querySelector('[data-testid="review-negative-text"]');
+            const dateEl = card.querySelector('[data-testid="review-date"]');
+            const flagImg = card.querySelector('[data-testid="review-avatar"] .fff1944c52 img[alt], [data-testid="review-country"] img[alt], img.b8d1620349');
+            const countrySpan = card.querySelector('[data-testid="review-avatar"] .fff1944c52 span:not(.cec4c9cfef), [data-testid="review-country"]');
+
+            const title = titleEl ? titleEl.innerText.trim() : '';
+            const positive = posEl ? posEl.innerText.trim().replace(/^„|“$/g, '').trim() : '';
+            const negative = negEl ? negEl.innerText.trim().replace(/^„|“$/g, '').trim() : '';
+            const dateText = dateEl ? dateEl.innerText.trim() : 'Aktuell';
+            const country = (flagImg ? flagImg.getAttribute('alt') : (countrySpan ? countrySpan.innerText : '')).trim();
+
+            const cardText = card.innerText || '';
             const yearMatch = cardText.match(/\b(20\d{2})\b/);
             if (yearMatch) {
               const reviewYear = parseInt(yearMatch[1], 10);
-              if (currentYear - reviewYear > 2) continue;
+              if (currentYear - reviewYear > 1) {
+                continue;
+              }
             }
-
-            const titleEl = card.querySelector('[data-testid="review-title"], h4');
-            const posEl = card.querySelector('[data-testid="review-positive-text"], .review_pos');
-            const negEl = card.querySelector('[data-testid="review-negative-text"], .review_neg');
-            const dateEl = card.querySelector('[data-testid="review-date"]');
-
-            const title = titleEl ? titleEl.innerText.trim() : '';
-            const positive = posEl ? posEl.innerText.trim() : '';
-            const negative = negEl ? negEl.innerText.trim() : '';
-            const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell');
 
             let fallbackText = '';
-            if (!positive && !negative) fallbackText = cardText.replace(title, '').substring(0, 300).trim();
+            if (!positive && !negative) {
+              fallbackText = cardText.replace(title, '').substring(0, 300).trim();
+            }
 
             if (title || positive || negative || fallbackText) {
-              results.push({ title, positive: positive || fallbackText, negative, date: dateText });
+              items.push({ title, positive: positive || fallbackText, negative, date: dateText, country });
             }
           }
-          return results;
+          return items;
-        }, { westernCountries: WESTERN_COUNTRIES }).catch(() => []);
+        }).catch(err => {
+          console.error(`[Job ${job.id}] Evaluate error on page ${pageCount}:`, err.message);
+          return [];
+        });
+
+        console.log(`[Job ${job.id}] 🔍 Echte Gästekarten auf Seite ${pageCount} gefunden: ${pageReviews.length}`);
 
         for (const rev of pageReviews) {
+          if (rev.country) {
+            const isWestern = WESTERN_COUNTRIES.some(c => rev.country.toLowerCase().includes(c.toLowerCase()));
+            if (!isWestern) {
+              console.log(`[Job ${job.id}]   └─ ❌ Land nicht in Liste: "${rev.country}"`);
+              continue;
+            }
+          }
           const key = `${rev.title}-${rev.positive}-${rev.date}`;
           if (!seenKeys.has(key) && extractedReviews.length < 80) {
             seenKeys.add(key);
             extractedReviews.push(rev);
+            console.log(`[Job ${job.id}]   └─ ✅ Akzeptiert: "${rev.title || rev.positive.substring(0, 30)}" (${rev.country || 'International'})`);
           }
         }
 
         if (extractedReviews.length >= 80) break;
 
-        try {
+        // Weiterblättern im Modal
-          const nextBtn = await page.$('#reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"]');
+        const nextResult = await page.evaluate(() => {
-          if (nextBtn) {
+          const nextBtn = document.querySelector('div[role="dialog"] button[aria-label="Nächste Seite"], button[aria-label="Nächste Seite"], button[aria-label*="Nächste Seite"]');
-            const isDisabled = await nextBtn.getAttribute('disabled');
+          if (nextBtn && !nextBtn.disabled && !nextBtn.hasAttribute('disabled')) {
-            if (isDisabled !== null) break;
+            nextBtn.scrollIntoView({ behavior: 'instant', block: 'center' });
-            await nextBtn.click();
+            nextBtn.click();
-            await page.waitForTimeout(2000);
+            nextBtn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
-            pageCount++;
+            return true;
-          } else break;
-        } catch (e) { break; }
           }
-      console.log(`[Job ${job.id}] ${extractedReviews.length} Kommentare extrahiert.`);
+          return false;
+        }).catch(() => false);
+
+        if (nextResult) {
+          await page.waitForTimeout(3000);
+        } else {
+          break;
+        }
+      }
+
+      console.log(`[Job ${job.id}] ✅ Insgesamt ${extractedReviews.length} passende Kommentare extrahiert.`);
     }
 
   } catch (error) {
-    console.error(`[Job ${job.id}] Fehler beim Scraping:`, error.message);
+    console.error(`[Job ${job.id}] ❌ Fehler beim Scraping:`, error.message);
   } finally {
     if (browser) await browser.close();
 
-// 4. Google Maps Fotos nur bei gezieltem KI-Review-Scan abrufen (spart SerpApi-Credits)
+    // 5. GOOGLE MAPS FOTOS NUR BEI KI-REVIEW SCAN ABRUFEN
     if (isReviewScan && hotelDetails.name) {
       try {
         bathroomImages = await getGoogleBathroomPhotosViaApi(hotelDetails.name, hotelDetails.address);
@@ -324,7 +718,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
       }
     }
 
-    // 5. Preishistorie und Bad-Bilder an Travel-App Webhook senden
+    // 6. PREISHISTORIE + STAMMDATEN AN NEXT.JS WEBHOOK SENDEN
     if (appWebhookUrl) {
       try {
         await axios.post(appWebhookUrl, {
@@ -342,13 +736,13 @@ const worker = new Worker('scrape-jobs', async (job) => {
           persons: domData.dPersons || 2,
           bathroomImages: Array.isArray(bathroomImages) && bathroomImages.length > 0 ? bathroomImages : undefined
         });
-        console.log(`[Job ${job.id}] ✅ Preishistorie in Postgres gespeichert!`);
+        console.log(`[Job ${job.id}] ✅ Preishistorie & Stammdaten in Postgres gespeichert!`);
       } catch (e) {
-        console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie:`, e.message);
+        console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie-Webhook:`, e.message);
       }
     }
 
-    // 6. Reviews an n8n senden
+    // 7. REVIEWS AN N8N WEBHOOK SENDEN
     if (isReviewScan && extractedReviews.length > 0) {
       try {
         await axios.post(webhookUrl, {
@@ -359,7 +753,9 @@ const worker = new Worker('scrape-jobs', async (job) => {
           timestamp: new Date().toISOString()
         });
         console.log(`[Job ${job.id}] 🚀 ${extractedReviews.length} Reviews an n8n gesendet!`);
-      } catch (e) {}
+      } catch (e) {
+        console.error(`[Job ${job.id}] ❌ Fehler beim Senden an n8n:`, e.message);
+      }
     }
   }
 }, { ...REDIS_CONFIG, concurrency: 2 });