diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2b7d3d2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +node_modules/ +.env +.env.local +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Generated debug dumps +src/*.html +src/*.png +debug/ + +.DS_Store diff --git a/README.md b/README.md index e69de29..493e994 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,54 @@ +# 🕵️ Scraper Service + +A robust background scraping service built with **Playwright**, **BullMQ**, and **Chromium** to scrape hotel metadata, real-time prices, Genius discounts, and guest reviews from Booking.com. + +--- + +## 🌟 Key Capabilities + +- **Deep Schema & Metadata Extraction:** + - Unpacks nested Schema.org JSON-LD scripts (including `@graph` arrays and accommodation schemas: `Hotel`, `Resort`, `Apartment`, `Hostel`, `LodgingBusiness`, etc.). + - Multi-tier DOM and OpenGraph meta tag fallbacks for title, address, hero images, and ratings. + - Automatic URL slug parser (`extractHotelNameFromUrl`) as an ultimate fallback if client-side rendering is obstructed. +- **AWS WAF Challenge & Anti-Bot Evasion:** + - **URL Sanitization (`cleanBookingUrl`):** Automatically cleans tracking tokens and expired challenge parameters (`chal_t`, `force_referer`, `sid`, `srpvid`, `srepoch`). + - **Browser Fingerprint Masking:** Emulates standard desktop Chrome headers, `window.chrome` runtime objects, `navigator.plugins`, and German language headers. + - **Active WAF Challenge Resolution:** Automatically detects AWS WAF challenge pages and waits for client-side JavaScript token generation and redirection. +- **Overlay & Banner Handling:** + - Auto-dismisses OneTrust cookie banners, modal dialogs, and selection popups before evaluating DOM data. +- **Dual-Mode Scraping:** + - **Quick Mode (Stammdaten & Prices):** Scrapes name, star rating, review count, address, hero image, room size, and prices in ~3–5 seconds and posts directly to Next.js webhook without LLM overhead. + - **Full AI Review Mode:** Sorts reviews by newest first, pages through reviews, filters for western travelers from the last 2 years, pulls Google Maps bathroom photos via SerpApi, and sends reviews to n8n for LLM summarization. +- **Debugging & Inspection:** + - Automatically captures post-render rendered DOM (`src/hotel-debug.html`) and screenshots (`src/hotel-debug.png`) when `DEBUG_HTML=true` or when requested via `?debug=true`. + +--- + +## ⚙️ Configuration (`.env` / Docker Environment) + +| Variable | Description | Default | +|---|---|---| +| `REDIS_HOST` | Redis host or container name | `redis` | +| `REDIS_PORT` | Redis port | `6379` | +| `DEBUG_HTML` | Enable saving `hotel-debug.html` and `hotel-debug.png` | `true` | +| `SERPAPI_KEY` | (Optional) SerpApi key for Google Maps bathroom photos | `null` | + +--- + +## 🚀 Running with Docker Compose + +```bash +docker compose up -d --build +``` + +### Viewing Logs +```bash +docker logs -f scraper-app +``` + +### Inspecting Scraped DOM Debug Files +When debugging hotel pages, the rendered DOM and screenshot are saved directly to: +- `src/hotel-debug.html` +- `src/hotel-debug.png` +- `src/review-debug.png` + diff --git a/docker-compose.yml b/docker-compose.yml index d93dccf..80ba2dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,7 @@ services: - REDIS_HOST=redis - REDIS_PORT=6379 - SERPAPI_KEY=${SERPAPI_KEY} + - DEBUG_HTML=${DEBUG_HTML:-true} env_file: - .env volumes: diff --git a/src/index.js b/src/index.js index 76355ef..e624f08 100644 --- a/src/index.js +++ b/src/index.js @@ -1,6 +1,7 @@ import { Worker } from 'bullmq'; import { chromium } from 'playwright'; import axios from 'axios'; +import fs from 'node:fs/promises'; // ========================================== // KONFIGURATION & LISTEN @@ -12,6 +13,8 @@ const REDIS_CONFIG = { } }; +const DEBUG_HTML = process.env.DEBUG_HTML === 'true' || process.env.DEBUG_HTML === '1'; + const SERPAPI_KEY = process.env.SERPAPI_KEY || null; const WESTERN_COUNTRIES = [ @@ -91,12 +94,58 @@ async function getGoogleBathroomPhotosViaApi(hotelName, hotelAddress) { } } +// ========================================== +// HILFSFUNKTION: HOTELNAME AUS URL EXTRAHIEREN +// ========================================== +function extractHotelNameFromUrl(urlString) { + try { + const parsed = new URL(urlString); + const parts = parsed.pathname.split('/').filter(Boolean); + const lastPart = parts[parts.length - 1] || ''; + const cleanSlug = lastPart.replace(/\.[a-z]{2}\.html$/i, '').replace(/\.html$/i, ''); + if (cleanSlug && cleanSlug.length > 2 && cleanSlug !== 'hotel') { + return cleanSlug + .split('-') + .map(w => w.charAt(0).toUpperCase() + w.slice(1)) + .join(' '); + } + } catch (e) {} + return null; +} + +// ========================================== +// HILFSFUNKTION: URL VON TRACKING & CHALLENGE-PARAMETERN BEREINIGEN +// ========================================== +function cleanBookingUrl(rawUrl) { + try { + const parsed = new URL(rawUrl); + // Behalte nur wesentliche Buchungsparameter und entferne challenge tokens (chal_t, force_referer, etc.) + const allowedParams = ['checkin', 'checkout', 'group_adults', 'no_rooms', 'group_children', 'selected_currency', 'lang']; + const newParams = new URLSearchParams(); + + for (const param of allowedParams) { + const val = parsed.searchParams.get(param); + if (val) newParams.set(param, val); + } + + if (!newParams.has('lang')) newParams.set('lang', 'de'); + if (!newParams.has('selected_currency')) newParams.set('selected_currency', 'EUR'); + + parsed.search = newParams.toString(); + parsed.hash = ''; + return parsed.toString(); + } catch (e) { + return rawUrl; + } +} + // ========================================== // BULLMQ WORKER: SCRAPING & REVIEWS // ========================================== const worker = new Worker('scrape-jobs', async (job) => { - const { url, webhookUrl, appWebhookUrl, hotelId } = job.data; + const { url, webhookUrl, appWebhookUrl, hotelId, debug } = job.data; const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http')); + const isDebugMode = Boolean(debug || DEBUG_HTML); console.log(`\n==================================================`); console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews + Fotos)' : 'PREIS-SCAN (Schnell)'} für: ${url}`); @@ -127,23 +176,111 @@ const worker = new Worker('scrape-jobs', async (job) => { userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', viewport: { width: 1366, height: 900 }, locale: 'de-DE', - timezoneId: 'Europe/Berlin' + timezoneId: 'Europe/Berlin', + extraHTTPHeaders: { + 'Accept-Language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7', + 'Upgrade-Insecure-Requests': '1', + 'Sec-Ch-Ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"', + 'Sec-Ch-Ua-Mobile': '?0', + 'Sec-Ch-Ua-Platform': '"Windows"' + } }); const page = await context.newPage(); - // Verhindert Erkennung über navigator.webdriver + // Verhindert Erkennung über navigator.webdriver und emuliert Standard-Browser-Eigenschaften await page.addInitScript(() => { Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); + window.chrome = { + runtime: {}, + loadTimes: function() {}, + csi: function() {}, + app: {} + }; + Object.defineProperty(navigator, 'plugins', { + get: () => [1, 2, 3, 4, 5] + }); + Object.defineProperty(navigator, 'languages', { + get: () => ['de-DE', 'de', 'en-US', 'en'] + }); }); - let targetUrl = url; + let targetUrl = cleanBookingUrl(url); if (isReviewScan && !targetUrl.includes('#tab-reviews')) { targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews'; } - await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 40000 }); - await page.waitForTimeout(2500); + console.log(`[Job ${job.id}] 🌐 Bereinigte Ziel-URL: ${targetUrl}`); + + await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 45000 }); + await page.waitForTimeout(2000); + + // Prüfen, ob AWS WAF Challenge aktiv ist + const isWafChallenge = await page.evaluate(() => { + return Boolean( + document.querySelector('#challenge-container') || + document.querySelector('script[src*="challenge.js"]') || + window.AwsWafIntegration || + (document.title === '' && (document.body ? document.body.innerText.includes('verify that you') : false)) + ); + }); + + if (isWafChallenge) { + console.log(`[Job ${job.id}] 🛡️ AWS WAF Challenge erkannt. Warte auf automatische Token-Weiterleitung...`); + try { + await page.waitForFunction(() => { + const hasChallenge = document.querySelector('#challenge-container') || document.querySelector('script[src*="challenge.js"]'); + const hasContent = document.querySelector('h1, h2, #hp_hotel_name, [data-testid="header-title"], script[type="application/ld+json"]'); + return !hasChallenge || Boolean(hasContent); + }, { timeout: 25000 }); + console.log(`[Job ${job.id}] 🔓 Weiterleitung nach AWS WAF erfolgt!`); + await page.waitForTimeout(2000); + } catch (wafErr) { + console.log(`[Job ${job.id}] ⚠️ AWS WAF Timeout. Versuche direkten Reload der Basis-URL...`); + try { + const parsed = new URL(targetUrl); + await page.goto(parsed.origin + parsed.pathname, { waitUntil: 'domcontentloaded', timeout: 30000 }); + await page.waitForTimeout(3000); + } catch (reloadErr) {} + } + } + + // Warte kurz auf wichtige Hotel-Elemente + try { + await page.waitForSelector('h1, h2, #hp_hotel_name, [data-testid="header-title"], script[type="application/ld+json"]', { timeout: 8000 }); + } catch (e) {} + + // Cookie-Banner & störende Overlays für alle Scans sofort schließen + try { + const dismissSelectors = [ + '#onetrust-accept-btn-handler', + 'button#onetrust-reject-all-handler', + 'button[data-testid="selection-modal-close-btn"]', + 'button[aria-label="Schließen"]', + 'button[aria-label="Close"]', + 'button.modal-mask-closeBtn' + ]; + for (const btnSel of dismissSelectors) { + const btn = await page.$(btnSel); + if (btn) { + await btn.click().catch(() => {}); + await page.waitForTimeout(300); + } + } + } catch (e) {} + + // HTML-Dump & Screenshot bei Debug-Modus sichern + if (isDebugMode) { + try { + const htmlContent = await page.content(); + await fs.writeFile('src/hotel-debug.html', htmlContent, 'utf-8'); + await page.screenshot({ path: 'src/hotel-debug.png', fullPage: false }); + console.log(`[Job ${job.id}] 📝 Debug-HTML gespeichert unter: scraper-service/src/hotel-debug.html`); + console.log(`[Job ${job.id}] 📸 Debug-Screenshot gespeichert unter: scraper-service/src/hotel-debug.png`); + } catch (err) { + console.error(`[Job ${job.id}] ⚠️ Fehler beim Speichern der Debug-Dateien:`, err.message); + } + } // DATUM AUS URL PARSEN try { @@ -152,28 +289,174 @@ const worker = new Worker('scrape-jobs', async (job) => { domData.checkoutDate = parsedUrl.searchParams.get('checkout'); } catch (e) {} - // 2. JSON-LD STAMMDATEN EXTRAHIEREN + // 2. STAMMDATEN EXTRAHIEREN (JSON-LD + DOM + META FALLBACKS) hotelDetails = await page.evaluate(() => { + let name = null; + let rating = null; + let reviewCount = null; + let address = null; + let image = null; + + // A. JSON-LD durchsuchen (inkl. @graph und verschiedenen Hotel-Typen) try { const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]')); for (const script of scripts) { try { - const json = JSON.parse(script.innerText || '{}'); - if (json['@type'] === 'Hotel' || json['@type'] === 'LodgingBusiness' || json.name) { - return { - name: json.name || null, - rating: json.aggregateRating?.ratingValue || null, - reviewCount: json.aggregateRating?.reviewCount || null, - address: json.address?.streetAddress || json.address?.addressLocality || null, - image: Array.isArray(json.image) ? json.image[0] : (json.image || null) - }; + const raw = JSON.parse(script.textContent || script.innerText || '{}'); + const items = Array.isArray(raw) ? raw : (raw['@graph'] ? raw['@graph'] : [raw]); + + for (const json of items) { + const type = (json['@type'] || '').toLowerCase(); + const isAccommodation = ['hotel', 'lodgingbusiness', 'resort', 'apartment', 'hostel', 'motel', 'bedandbreakfast', 'guesthouse', 'inn', 'accommodation'].some(t => type.includes(t)); + + if (isAccommodation || (json.name && (json.address || json.aggregateRating))) { + if (!name && json.name && typeof json.name === 'string') { + name = json.name.trim(); + } + if (!rating && json.aggregateRating?.ratingValue) { + const r = parseFloat(String(json.aggregateRating.ratingValue).replace(',', '.')); + if (!isNaN(r)) rating = r; + } + if (!reviewCount && json.aggregateRating?.reviewCount) { + const rc = parseInt(String(json.aggregateRating.reviewCount).replace(/\D/g, ''), 10); + if (!isNaN(rc)) reviewCount = rc; + } + if (!address && json.address) { + if (typeof json.address === 'string') { + address = json.address.trim(); + } else if (typeof json.address === 'object') { + const parts = [ + json.address.streetAddress, + json.address.postalCode, + json.address.addressLocality, + json.address.addressCountry + ].filter(Boolean); + if (parts.length > 0) address = parts.join(', ').trim(); + } + } + if (!image && json.image) { + if (Array.isArray(json.image) && json.image.length > 0) { + image = typeof json.image[0] === 'string' ? json.image[0] : json.image[0]?.url; + } else if (typeof json.image === 'string') { + image = json.image; + } else if (json.image?.url) { + image = json.image.url; + } + } + } } } catch (e) {} } } catch (e) {} - return { name: null, rating: null, reviewCount: null, address: null, image: null }; + + // B. DOM & META FALLBACKS + + // 1. Name Fallback + if (!name) { + const nameSelectors = [ + 'h2.pp-header__title', + '[data-testid="header-title"]', + '#hp_hotel_name', + 'h2.hp__hotel-name', + 'h2.d2fee87262', + '.hp__hotel-title h2', + 'h1[data-testid="header-title"]', + 'h1' + ]; + for (const sel of nameSelectors) { + const el = document.querySelector(sel); + if (el && el.innerText) { + const rawText = el.innerText.split('\n')[0].replace(/^(Hotel|Resort|Apartment)\s+/i, '').trim(); + if (rawText && rawText.length > 2 && !rawText.toLowerCase().includes('booking.com')) { + name = rawText; + break; + } + } + } + + // Meta Tag Fallback für Name + if (!name) { + const ogTitle = document.querySelector('meta[property="og:title"]')?.getAttribute('content'); + if (ogTitle) { + name = ogTitle.split('–')[0].split('-')[0].split('|')[0].replace(/^(Hotel|Resort|Apartment)\s+/i, '').trim(); + } else if (document.title) { + name = document.title.split('–')[0].split('-')[0].split('|')[0].replace(/^(Hotel|Resort|Apartment)\s+/i, '').trim(); + } + } + } + + // 2. Adresse Fallback + if (!address) { + const addrSelectors = [ + '[data-testid="Address-wrapper"] span', + 'span.hp_address_subtitle', + '[data-node_tt_id="location_score_tooltip"]', + '.bui-link--light[data-source="top_link"]', + 'span[data-testid="address-line"]' + ]; + for (const sel of addrSelectors) { + const el = document.querySelector(sel); + if (el && el.innerText && el.innerText.trim().length > 5) { + address = el.innerText.trim().replace(/\s+/g, ' '); + break; + } + } + } + + // 3. Rating & ReviewCount Fallback + if (!rating || !reviewCount) { + try { + const scoreBlock = document.querySelector('[data-testid="review-score-right-component"], .b5cd09854e, .d8eab2cf7f, [data-testid="review-score-component"]'); + const scoreText = (scoreBlock ? scoreBlock.innerText : (document.body ? document.body.innerText : '')) || ''; + + if (!rating && scoreText) { + const ratingMatch = scoreText.match(/(?:Bewertet mit|Scored|Punkte|Note|Score)?\s*([0-9][0-9]?[.,][0-9])/i); + if (ratingMatch) { + const r = parseFloat(ratingMatch[1].replace(',', '.')); + if (!isNaN(r) && r <= 10) rating = r; + } + } + + if (!reviewCount && scoreText) { + const countMatch = scoreText.match(/([0-9][0-9.,]*)\s*(?:Bewertungen|reviews|rezensionen|avis|opiniones|Guest reviews)/i); + if (countMatch) { + const rc = parseInt(countMatch[1].replace(/[.,]/g, ''), 10); + if (!isNaN(rc)) reviewCount = rc; + } + } + } catch (e) {} + } + + // 4. Hero-Image Fallback + if (!image) { + const ogImage = document.querySelector('meta[property="og:image"]')?.getAttribute('content'); + if (ogImage && ogImage.startsWith('http')) { + image = ogImage; + } else { + const imgSelectors = [ + '[data-testid="hotel-gallery"] img', + '.gallery_image_round_corners img', + '#hotel_main_content img', + 'img.bh-photo-grid-thumb' + ]; + for (const sel of imgSelectors) { + const imgEl = document.querySelector(sel); + if (imgEl && imgEl.getAttribute('src') && imgEl.getAttribute('src').startsWith('http')) { + image = imgEl.getAttribute('src'); + break; + } + } + } + } + + return { name, rating, reviewCount, address, image }; }); + // Fallback: Name aus URL-Slug ableiten wenn im HTML nichts gefunden wurde + if (!hotelDetails.name) { + hotelDetails.name = extractHotelNameFromUrl(url); + } + // 3. DOM-DATEN (Nächte, Personen, Zimmergröße) const extraDom = await page.evaluate(() => { try { @@ -197,30 +480,58 @@ const worker = new Worker('scrape-jobs', async (job) => { // PREIS AUSLESEN scrapedPrice = await page.evaluate(() => { - const selectors = [ - '[data-testid="price-and-discounted-price"]', - '.totalPrice .bui-price-display__value', - '.hprt-price-price', - '.prco-valign-middle-helper' - ]; - for (const sel of selectors) { - const el = document.querySelector(sel); - if (el && el.innerText && /\d/.test(el.innerText)) return el.innerText.trim(); - } + try { + const selectors = [ + '[data-testid="price-and-discounted-price"]', + '.totalPrice .bui-price-display__value', + '.hprt-price-price', + '.prco-valign-middle-helper', + '[data-testid="price-display"]', + 'span.prco-inline-block-maker-helper', + '.bui-price-display__value' + ]; + for (const sel of selectors) { + const el = document.querySelector(sel); + if (el && el.innerText && /\d/.test(el.innerText)) return el.innerText.trim(); + } + + const bodyText = document.body ? (document.body.innerText || '') : ''; + const priceMatch = bodyText.match(/(?:€|EUR)\s*([0-9][0-9.,]*)|([0-9][0-9.,]*)\s*(?:€|EUR)/i); + if (priceMatch) { + return (priceMatch[1] || priceMatch[2] || '').trim(); + } + } catch (e) {} return null; }); // GENIUS BADGE PRÜFUNG hasGenius = await page.evaluate(() => { - const geniusSelectors = [ - '.m-badge__genius', - 'svg.genius-badge', - '.-genius-new_identity-genius_badge', - '.c-deals-container__genius-badge-fix', - '[aria-label*="Genius"]', - '[data-testid="deal-badge"]' - ]; - return geniusSelectors.some(sel => Boolean(document.querySelector(sel))); + try { + const geniusSelectors = [ + '.m-badge__genius', + 'svg.genius-badge', + '.-genius-new_identity-genius_badge', + '.c-deals-container__genius-badge-fix', + '[aria-label*="Genius"]', + '[data-testid="deal-badge"]' + ]; + return geniusSelectors.some(sel => Boolean(document.querySelector(sel))); + } catch (e) { + return false; + } + }); + + console.log(`[Job ${job.id}] 📋 Extrahierte Stammdaten:`, { + name: hotelDetails.name, + rating: hotelDetails.rating, + reviewCount: hotelDetails.reviewCount, + address: hotelDetails.address, + image: hotelDetails.image ? (hotelDetails.image.substring(0, 60) + '...') : null, + price: scrapedPrice, + hasGenius, + nights: domData.dNights, + persons: domData.dPersons, + roomSizeM2 }); if (hasGenius) { diff --git a/src/review-debug.png b/src/review-debug.png index d38c9e6..46ab8ca 100644 Binary files a/src/review-debug.png and b/src/review-debug.png differ