fix: resolve AWS WAF challenge, add URL sanitization, robust DOM/JSON-LD fallback extraction, and update README

.gitignore

@@ -0,0 +1,13 @@
+node_modules/
+.env
+.env.local
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Generated debug dumps
+src/*.html
+src/*.png
+debug/
+
+.DS_Store

README.md

@@ -0,0 +1,54 @@
+# 🕵️ Scraper Service
+
+A robust background scraping service built with **Playwright**, **BullMQ**, and **Chromium** to scrape hotel metadata, real-time prices, Genius discounts, and guest reviews from Booking.com.
+
+---
+
+## 🌟 Key Capabilities
+
+- **Deep Schema & Metadata Extraction:**
+  - Unpacks nested Schema.org JSON-LD scripts (including `@graph` arrays and accommodation schemas: `Hotel`, `Resort`, `Apartment`, `Hostel`, `LodgingBusiness`, etc.).
+  - Multi-tier DOM and OpenGraph meta tag fallbacks for title, address, hero images, and ratings.
+  - Automatic URL slug parser (`extractHotelNameFromUrl`) as an ultimate fallback if client-side rendering is obstructed.
+- **AWS WAF Challenge & Anti-Bot Evasion:**
+  - **URL Sanitization (`cleanBookingUrl`):** Automatically cleans tracking tokens and expired challenge parameters (`chal_t`, `force_referer`, `sid`, `srpvid`, `srepoch`).
+  - **Browser Fingerprint Masking:** Emulates standard desktop Chrome headers, `window.chrome` runtime objects, `navigator.plugins`, and German language headers.
+  - **Active WAF Challenge Resolution:** Automatically detects AWS WAF challenge pages and waits for client-side JavaScript token generation and redirection.
+- **Overlay & Banner Handling:**
+  - Auto-dismisses OneTrust cookie banners, modal dialogs, and selection popups before evaluating DOM data.
+- **Dual-Mode Scraping:**
+  - **Quick Mode (Stammdaten & Prices):** Scrapes name, star rating, review count, address, hero image, room size, and prices in ~3–5 seconds and posts directly to Next.js webhook without LLM overhead.
+  - **Full AI Review Mode:** Sorts reviews by newest first, pages through reviews, filters for western travelers from the last 2 years, pulls Google Maps bathroom photos via SerpApi, and sends reviews to n8n for LLM summarization.
+- **Debugging & Inspection:**
+  - Automatically captures post-render rendered DOM (`src/hotel-debug.html`) and screenshots (`src/hotel-debug.png`) when `DEBUG_HTML=true` or when requested via `?debug=true`.
+
+---
+
+## ⚙️ Configuration (`.env` / Docker Environment)
+
+| Variable | Description | Default |
+|---|---|---|
+| `REDIS_HOST` | Redis host or container name | `redis` |
+| `REDIS_PORT` | Redis port | `6379` |
+| `DEBUG_HTML` | Enable saving `hotel-debug.html` and `hotel-debug.png` | `true` |
+| `SERPAPI_KEY` | (Optional) SerpApi key for Google Maps bathroom photos | `null` |
+
+---
+
+## 🚀 Running with Docker Compose
+
+```bash
+docker compose up -d --build
+```
+
+### Viewing Logs
+```bash
+docker logs -f scraper-app
+```
+
+### Inspecting Scraped DOM Debug Files
+When debugging hotel pages, the rendered DOM and screenshot are saved directly to:
+- `src/hotel-debug.html`
+- `src/hotel-debug.png`
+- `src/review-debug.png`
+

docker-compose.yml

@@ -17,6 +17,7 @@ services:
       - REDIS_HOST=redis
       - REDIS_PORT=6379
       - SERPAPI_KEY=${SERPAPI_KEY}
+      - DEBUG_HTML=${DEBUG_HTML:-true}
     env_file:
       - .env
     volumes:

src/index.js

@@ -1,6 +1,7 @@
 import { Worker } from 'bullmq';
 import { chromium } from 'playwright';
 import axios from 'axios';
+import fs from 'node:fs/promises';
 
 // ==========================================
 // KONFIGURATION & LISTEN
@@ -12,6 +13,8 @@ const REDIS_CONFIG = {
   }
 };
 
+const DEBUG_HTML = process.env.DEBUG_HTML === 'true' || process.env.DEBUG_HTML === '1';
+
 const SERPAPI_KEY = process.env.SERPAPI_KEY || null;
 
 const WESTERN_COUNTRIES = [
@@ -91,12 +94,58 @@ async function getGoogleBathroomPhotosViaApi(hotelName, hotelAddress) {
   }
 }
 
+// ==========================================
+// HILFSFUNKTION: HOTELNAME AUS URL EXTRAHIEREN
+// ==========================================
+function extractHotelNameFromUrl(urlString) {
+  try {
+    const parsed = new URL(urlString);
+    const parts = parsed.pathname.split('/').filter(Boolean);
+    const lastPart = parts[parts.length - 1] || '';
+    const cleanSlug = lastPart.replace(/\.[a-z]{2}\.html$/i, '').replace(/\.html$/i, '');
+    if (cleanSlug && cleanSlug.length > 2 && cleanSlug !== 'hotel') {
+      return cleanSlug
+        .split('-')
+        .map(w => w.charAt(0).toUpperCase() + w.slice(1))
+        .join(' ');
+    }
+  } catch (e) {}
+  return null;
+}
+
+// ==========================================
+// HILFSFUNKTION: URL VON TRACKING & CHALLENGE-PARAMETERN BEREINIGEN
+// ==========================================
+function cleanBookingUrl(rawUrl) {
+  try {
+    const parsed = new URL(rawUrl);
+    // Behalte nur wesentliche Buchungsparameter und entferne challenge tokens (chal_t, force_referer, etc.)
+    const allowedParams = ['checkin', 'checkout', 'group_adults', 'no_rooms', 'group_children', 'selected_currency', 'lang'];
+    const newParams = new URLSearchParams();
+
+    for (const param of allowedParams) {
+      const val = parsed.searchParams.get(param);
+      if (val) newParams.set(param, val);
+    }
+
+    if (!newParams.has('lang')) newParams.set('lang', 'de');
+    if (!newParams.has('selected_currency')) newParams.set('selected_currency', 'EUR');
+
+    parsed.search = newParams.toString();
+    parsed.hash = '';
+    return parsed.toString();
+  } catch (e) {
+    return rawUrl;
+  }
+}
+
 // ==========================================
 // BULLMQ WORKER: SCRAPING & REVIEWS
 // ==========================================
 const worker = new Worker('scrape-jobs', async (job) => {
-  const { url, webhookUrl, appWebhookUrl, hotelId } = job.data;
+  const { url, webhookUrl, appWebhookUrl, hotelId, debug } = job.data;
   const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http'));
+  const isDebugMode = Boolean(debug || DEBUG_HTML);
   
   console.log(`\n==================================================`);
   console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews + Fotos)' : 'PREIS-SCAN (Schnell)'} für: ${url}`);
@@ -127,23 +176,111 @@ const worker = new Worker('scrape-jobs', async (job) => {
       userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
       viewport: { width: 1366, height: 900 },
       locale: 'de-DE',
-      timezoneId: 'Europe/Berlin'
+      timezoneId: 'Europe/Berlin',
+      extraHTTPHeaders: {
+        'Accept-Language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
+        'Upgrade-Insecure-Requests': '1',
+        'Sec-Ch-Ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"',
+        'Sec-Ch-Ua-Mobile': '?0',
+        'Sec-Ch-Ua-Platform': '"Windows"'
+      }
     });
 
     const page = await context.newPage();
 
-    // Verhindert Erkennung über navigator.webdriver
+    // Verhindert Erkennung über navigator.webdriver und emuliert Standard-Browser-Eigenschaften
     await page.addInitScript(() => {
       Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+      window.chrome = {
+        runtime: {},
+        loadTimes: function() {},
+        csi: function() {},
+        app: {}
+      };
+      Object.defineProperty(navigator, 'plugins', {
+        get: () => [1, 2, 3, 4, 5]
+      });
+      Object.defineProperty(navigator, 'languages', {
+        get: () => ['de-DE', 'de', 'en-US', 'en']
+      });
     });
 
-    let targetUrl = url;
+    let targetUrl = cleanBookingUrl(url);
     if (isReviewScan && !targetUrl.includes('#tab-reviews')) {
       targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews';
     }
 
-    await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 40000 });
-    await page.waitForTimeout(2500);
+    console.log(`[Job ${job.id}] 🌐 Bereinigte Ziel-URL: ${targetUrl}`);
+
+    await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 45000 });
+    await page.waitForTimeout(2000);
+
+    // Prüfen, ob AWS WAF Challenge aktiv ist
+    const isWafChallenge = await page.evaluate(() => {
+      return Boolean(
+        document.querySelector('#challenge-container') ||
+        document.querySelector('script[src*="challenge.js"]') ||
+        window.AwsWafIntegration ||
+        (document.title === '' && (document.body ? document.body.innerText.includes('verify that you') : false))
+      );
+    });
+
+    if (isWafChallenge) {
+      console.log(`[Job ${job.id}] 🛡️ AWS WAF Challenge erkannt. Warte auf automatische Token-Weiterleitung...`);
+      try {
+        await page.waitForFunction(() => {
+          const hasChallenge = document.querySelector('#challenge-container') || document.querySelector('script[src*="challenge.js"]');
+          const hasContent = document.querySelector('h1, h2, #hp_hotel_name, [data-testid="header-title"], script[type="application/ld+json"]');
+          return !hasChallenge || Boolean(hasContent);
+        }, { timeout: 25000 });
+        console.log(`[Job ${job.id}] 🔓 Weiterleitung nach AWS WAF erfolgt!`);
+        await page.waitForTimeout(2000);
+      } catch (wafErr) {
+        console.log(`[Job ${job.id}] ⚠️ AWS WAF Timeout. Versuche direkten Reload der Basis-URL...`);
+        try {
+          const parsed = new URL(targetUrl);
+          await page.goto(parsed.origin + parsed.pathname, { waitUntil: 'domcontentloaded', timeout: 30000 });
+          await page.waitForTimeout(3000);
+        } catch (reloadErr) {}
+      }
+    }
+
+    // Warte kurz auf wichtige Hotel-Elemente
+    try {
+      await page.waitForSelector('h1, h2, #hp_hotel_name, [data-testid="header-title"], script[type="application/ld+json"]', { timeout: 8000 });
+    } catch (e) {}
+
+    // Cookie-Banner & störende Overlays für alle Scans sofort schließen
+    try {
+      const dismissSelectors = [
+        '#onetrust-accept-btn-handler',
+        'button#onetrust-reject-all-handler',
+        'button[data-testid="selection-modal-close-btn"]',
+        'button[aria-label="Schließen"]',
+        'button[aria-label="Close"]',
+        'button.modal-mask-closeBtn'
+      ];
+      for (const btnSel of dismissSelectors) {
+        const btn = await page.$(btnSel);
+        if (btn) {
+          await btn.click().catch(() => {});
+          await page.waitForTimeout(300);
+        }
+      }
+    } catch (e) {}
+
+    // HTML-Dump & Screenshot bei Debug-Modus sichern
+    if (isDebugMode) {
+      try {
+        const htmlContent = await page.content();
+        await fs.writeFile('src/hotel-debug.html', htmlContent, 'utf-8');
+        await page.screenshot({ path: 'src/hotel-debug.png', fullPage: false });
+        console.log(`[Job ${job.id}] 📝 Debug-HTML gespeichert unter: scraper-service/src/hotel-debug.html`);
+        console.log(`[Job ${job.id}] 📸 Debug-Screenshot gespeichert unter: scraper-service/src/hotel-debug.png`);
+      } catch (err) {
+        console.error(`[Job ${job.id}] ⚠️ Fehler beim Speichern der Debug-Dateien:`, err.message);
+      }
+    }
 
     // DATUM AUS URL PARSEN
     try {
@@ -152,28 +289,174 @@ const worker = new Worker('scrape-jobs', async (job) => {
       domData.checkoutDate = parsedUrl.searchParams.get('checkout');
     } catch (e) {}
 
-    // 2. JSON-LD STAMMDATEN EXTRAHIEREN
+    // 2. STAMMDATEN EXTRAHIEREN (JSON-LD + DOM + META FALLBACKS)
     hotelDetails = await page.evaluate(() => {
+      let name = null;
+      let rating = null;
+      let reviewCount = null;
+      let address = null;
+      let image = null;
+
+      // A. JSON-LD durchsuchen (inkl. @graph und verschiedenen Hotel-Typen)
       try {
         const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
         for (const script of scripts) {
           try {
-            const json = JSON.parse(script.innerText || '{}');
-            if (json['@type'] === 'Hotel' || json['@type'] === 'LodgingBusiness' || json.name) {
-              return {
-                name: json.name || null,
-                rating: json.aggregateRating?.ratingValue || null,
-                reviewCount: json.aggregateRating?.reviewCount || null,
-                address: json.address?.streetAddress || json.address?.addressLocality || null,
-                image: Array.isArray(json.image) ? json.image[0] : (json.image || null)
-              };
+            const raw = JSON.parse(script.textContent || script.innerText || '{}');
+            const items = Array.isArray(raw) ? raw : (raw['@graph'] ? raw['@graph'] : [raw]);
+
+            for (const json of items) {
+              const type = (json['@type'] || '').toLowerCase();
+              const isAccommodation = ['hotel', 'lodgingbusiness', 'resort', 'apartment', 'hostel', 'motel', 'bedandbreakfast', 'guesthouse', 'inn', 'accommodation'].some(t => type.includes(t));
+
+              if (isAccommodation || (json.name && (json.address || json.aggregateRating))) {
+                if (!name && json.name && typeof json.name === 'string') {
+                  name = json.name.trim();
+                }
+                if (!rating && json.aggregateRating?.ratingValue) {
+                  const r = parseFloat(String(json.aggregateRating.ratingValue).replace(',', '.'));
+                  if (!isNaN(r)) rating = r;
+                }
+                if (!reviewCount && json.aggregateRating?.reviewCount) {
+                  const rc = parseInt(String(json.aggregateRating.reviewCount).replace(/\D/g, ''), 10);
+                  if (!isNaN(rc)) reviewCount = rc;
+                }
+                if (!address && json.address) {
+                  if (typeof json.address === 'string') {
+                    address = json.address.trim();
+                  } else if (typeof json.address === 'object') {
+                    const parts = [
+                      json.address.streetAddress,
+                      json.address.postalCode,
+                      json.address.addressLocality,
+                      json.address.addressCountry
+                    ].filter(Boolean);
+                    if (parts.length > 0) address = parts.join(', ').trim();
+                  }
+                }
+                if (!image && json.image) {
+                  if (Array.isArray(json.image) && json.image.length > 0) {
+                    image = typeof json.image[0] === 'string' ? json.image[0] : json.image[0]?.url;
+                  } else if (typeof json.image === 'string') {
+                    image = json.image;
+                  } else if (json.image?.url) {
+                    image = json.image.url;
+                  }
+                }
+              }
             }
           } catch (e) {}
         }
       } catch (e) {}
-      return { name: null, rating: null, reviewCount: null, address: null, image: null };
+
+      // B. DOM & META FALLBACKS
+
+      // 1. Name Fallback
+      if (!name) {
+        const nameSelectors = [
+          'h2.pp-header__title',
+          '[data-testid="header-title"]',
+          '#hp_hotel_name',
+          'h2.hp__hotel-name',
+          'h2.d2fee87262',
+          '.hp__hotel-title h2',
+          'h1[data-testid="header-title"]',
+          'h1'
+        ];
+        for (const sel of nameSelectors) {
+          const el = document.querySelector(sel);
+          if (el && el.innerText) {
+            const rawText = el.innerText.split('\n')[0].replace(/^(Hotel|Resort|Apartment)\s+/i, '').trim();
+            if (rawText && rawText.length > 2 && !rawText.toLowerCase().includes('booking.com')) {
+              name = rawText;
+              break;
+            }
+          }
+        }
+
+        // Meta Tag Fallback für Name
+        if (!name) {
+          const ogTitle = document.querySelector('meta[property="og:title"]')?.getAttribute('content');
+          if (ogTitle) {
+            name = ogTitle.split('–')[0].split('-')[0].split('|')[0].replace(/^(Hotel|Resort|Apartment)\s+/i, '').trim();
+          } else if (document.title) {
+            name = document.title.split('–')[0].split('-')[0].split('|')[0].replace(/^(Hotel|Resort|Apartment)\s+/i, '').trim();
+          }
+        }
+      }
+
+      // 2. Adresse Fallback
+      if (!address) {
+        const addrSelectors = [
+          '[data-testid="Address-wrapper"] span',
+          'span.hp_address_subtitle',
+          '[data-node_tt_id="location_score_tooltip"]',
+          '.bui-link--light[data-source="top_link"]',
+          'span[data-testid="address-line"]'
+        ];
+        for (const sel of addrSelectors) {
+          const el = document.querySelector(sel);
+          if (el && el.innerText && el.innerText.trim().length > 5) {
+            address = el.innerText.trim().replace(/\s+/g, ' ');
+            break;
+          }
+        }
+      }
+
+      // 3. Rating & ReviewCount Fallback
+      if (!rating || !reviewCount) {
+        try {
+          const scoreBlock = document.querySelector('[data-testid="review-score-right-component"], .b5cd09854e, .d8eab2cf7f, [data-testid="review-score-component"]');
+          const scoreText = (scoreBlock ? scoreBlock.innerText : (document.body ? document.body.innerText : '')) || '';
+
+          if (!rating && scoreText) {
+            const ratingMatch = scoreText.match(/(?:Bewertet mit|Scored|Punkte|Note|Score)?\s*([0-9][0-9]?[.,][0-9])/i);
+            if (ratingMatch) {
+              const r = parseFloat(ratingMatch[1].replace(',', '.'));
+              if (!isNaN(r) && r <= 10) rating = r;
+            }
+          }
+
+          if (!reviewCount && scoreText) {
+            const countMatch = scoreText.match(/([0-9][0-9.,]*)\s*(?:Bewertungen|reviews|rezensionen|avis|opiniones|Guest reviews)/i);
+            if (countMatch) {
+              const rc = parseInt(countMatch[1].replace(/[.,]/g, ''), 10);
+              if (!isNaN(rc)) reviewCount = rc;
+            }
+          }
+        } catch (e) {}
+      }
+
+      // 4. Hero-Image Fallback
+      if (!image) {
+        const ogImage = document.querySelector('meta[property="og:image"]')?.getAttribute('content');
+        if (ogImage && ogImage.startsWith('http')) {
+          image = ogImage;
+        } else {
+          const imgSelectors = [
+            '[data-testid="hotel-gallery"] img',
+            '.gallery_image_round_corners img',
+            '#hotel_main_content img',
+            'img.bh-photo-grid-thumb'
+          ];
+          for (const sel of imgSelectors) {
+            const imgEl = document.querySelector(sel);
+            if (imgEl && imgEl.getAttribute('src') && imgEl.getAttribute('src').startsWith('http')) {
+              image = imgEl.getAttribute('src');
+              break;
+            }
+          }
+        }
+      }
+
+      return { name, rating, reviewCount, address, image };
     });
 
+    // Fallback: Name aus URL-Slug ableiten wenn im HTML nichts gefunden wurde
+    if (!hotelDetails.name) {
+      hotelDetails.name = extractHotelNameFromUrl(url);
+    }
+
     // 3. DOM-DATEN (Nächte, Personen, Zimmergröße)
     const extraDom = await page.evaluate(() => {
       try {
@@ -197,30 +480,58 @@ const worker = new Worker('scrape-jobs', async (job) => {
 
     // PREIS AUSLESEN
     scrapedPrice = await page.evaluate(() => {
-      const selectors = [
-        '[data-testid="price-and-discounted-price"]',
-        '.totalPrice .bui-price-display__value',
-        '.hprt-price-price',
-        '.prco-valign-middle-helper'
-      ];
-      for (const sel of selectors) {
-        const el = document.querySelector(sel);
-        if (el && el.innerText && /\d/.test(el.innerText)) return el.innerText.trim();
-      }
+      try {
+        const selectors = [
+          '[data-testid="price-and-discounted-price"]',
+          '.totalPrice .bui-price-display__value',
+          '.hprt-price-price',
+          '.prco-valign-middle-helper',
+          '[data-testid="price-display"]',
+          'span.prco-inline-block-maker-helper',
+          '.bui-price-display__value'
+        ];
+        for (const sel of selectors) {
+          const el = document.querySelector(sel);
+          if (el && el.innerText && /\d/.test(el.innerText)) return el.innerText.trim();
+        }
+
+        const bodyText = document.body ? (document.body.innerText || '') : '';
+        const priceMatch = bodyText.match(/(?:€|EUR)\s*([0-9][0-9.,]*)|([0-9][0-9.,]*)\s*(?:€|EUR)/i);
+        if (priceMatch) {
+          return (priceMatch[1] || priceMatch[2] || '').trim();
+        }
+      } catch (e) {}
       return null;
     });
 
     // GENIUS BADGE PRÜFUNG
     hasGenius = await page.evaluate(() => {
-      const geniusSelectors = [
-        '.m-badge__genius',
-        'svg.genius-badge',
-        '.-genius-new_identity-genius_badge',
-        '.c-deals-container__genius-badge-fix',
-        '[aria-label*="Genius"]',
-        '[data-testid="deal-badge"]'
-      ];
-      return geniusSelectors.some(sel => Boolean(document.querySelector(sel)));
+      try {
+        const geniusSelectors = [
+          '.m-badge__genius',
+          'svg.genius-badge',
+          '.-genius-new_identity-genius_badge',
+          '.c-deals-container__genius-badge-fix',
+          '[aria-label*="Genius"]',
+          '[data-testid="deal-badge"]'
+        ];
+        return geniusSelectors.some(sel => Boolean(document.querySelector(sel)));
+      } catch (e) {
+        return false;
+      }
+    });
+
+    console.log(`[Job ${job.id}] 📋 Extrahierte Stammdaten:`, {
+      name: hotelDetails.name,
+      rating: hotelDetails.rating,
+      reviewCount: hotelDetails.reviewCount,
+      address: hotelDetails.address,
+      image: hotelDetails.image ? (hotelDetails.image.substring(0, 60) + '...') : null,
+      price: scrapedPrice,
+      hasGenius,
+      nights: domData.dNights,
+      persons: domData.dPersons,
+      roomSizeM2
     });
 
     if (hasGenius) {