763 lines
31 KiB
JavaScript
763 lines
31 KiB
JavaScript
import { Worker } from 'bullmq';
|
||
import { chromium } from 'playwright';
|
||
import axios from 'axios';
|
||
import fs from 'node:fs/promises';
|
||
|
||
// ==========================================
|
||
// KONFIGURATION & LISTEN
|
||
// ==========================================
|
||
const REDIS_CONFIG = {
|
||
connection: {
|
||
host: process.env.REDIS_HOST || 'redis',
|
||
port: parseInt(process.env.REDIS_PORT || '6379', 10)
|
||
}
|
||
};
|
||
|
||
const DEBUG_HTML = process.env.DEBUG_HTML === 'true' || process.env.DEBUG_HTML === '1';
|
||
|
||
const SERPAPI_KEY = process.env.SERPAPI_KEY || null;
|
||
|
||
const WESTERN_COUNTRIES = [
|
||
'Deutschland', 'Schweiz', 'Österreich', 'Germany', 'Switzerland', 'Austria',
|
||
'Vereinigte Staaten', 'USA', 'United States', 'Vereinigtes Königreich', 'Großbritannien',
|
||
'United Kingdom', 'UK', 'Niederlande', 'Netherlands', 'Frankreich', 'France',
|
||
'Spanien', 'Spain', 'Italien', 'Italy', 'Kanada', 'Canada', 'Australien', 'Australia',
|
||
'Belgien', 'Belgium', 'Schweden', 'Sweden', 'Norwegen', 'Norway', 'Dänemark', 'Denmark',
|
||
'Finnland', 'Finland', 'Irland', 'Ireland', 'Luxemburg', 'Luxembourg',
|
||
'Neuseeland', 'New Zealand'
|
||
];
|
||
|
||
// ==========================================
|
||
// HILFSFUNKTION: GOOGLE MAPS BAD-FOTOS VIA SERPAPI
|
||
// ==========================================
|
||
async function getGoogleBathroomPhotosViaApi(hotelName, hotelAddress) {
|
||
if (!SERPAPI_KEY) {
|
||
console.log('ℹ️ [SerpApi] Kein SERPAPI_KEY konfiguriert – überspringe Google-Maps-Fotos.');
|
||
return [];
|
||
}
|
||
|
||
try {
|
||
let cleanLocation = '';
|
||
if (hotelAddress) {
|
||
const parts = hotelAddress.split(',').map(p => p.trim());
|
||
cleanLocation = parts.length > 1 ? parts.slice(-2).join(' ') : hotelAddress;
|
||
}
|
||
const query = `${hotelName} ${cleanLocation}`.trim();
|
||
console.log(`🔍 [SerpApi] Suche Google Maps Ort für: "${query}"...`);
|
||
|
||
const searchRes = await axios.get('https://serpapi.com/search.json', {
|
||
params: { engine: 'google_maps', q: query, hl: 'de', api_key: SERPAPI_KEY },
|
||
timeout: 15000
|
||
});
|
||
|
||
const placeResults = searchRes.data.place_results;
|
||
const localResults = searchRes.data.local_results;
|
||
const dataId = placeResults?.data_id || (localResults && localResults[0]?.data_id) || searchRes.data.search_metadata?.google_maps_data_id;
|
||
|
||
if (!dataId) {
|
||
console.log(`⚠️ [SerpApi] Keine Google Maps data_id für "${query}" gefunden.`);
|
||
return [];
|
||
}
|
||
|
||
const photosRes = await axios.get('https://serpapi.com/search.json', {
|
||
params: { engine: 'google_maps_photos', data_id: dataId, hl: 'de', api_key: SERPAPI_KEY },
|
||
timeout: 15000
|
||
});
|
||
|
||
const categories = photosRes.data.categories || [];
|
||
const strictBathKeywords = ['badezimmer', 'bad', 'bathroom', 'wc', 'toilet', 'washroom', 'restroom'];
|
||
const bathCat = categories.find(c => {
|
||
const title = (c.title || '').toLowerCase().trim();
|
||
return strictBathKeywords.some(kw => title.includes(kw));
|
||
});
|
||
|
||
let photosList = [];
|
||
if (bathCat?.id) {
|
||
console.log(`🚽 [SerpApi] Bad-Kategorie gefunden: "${bathCat.title}"`);
|
||
const catRes = await axios.get('https://serpapi.com/search.json', {
|
||
params: { engine: 'google_maps_photos', data_id: dataId, category_id: bathCat.id, hl: 'de', api_key: SERPAPI_KEY },
|
||
timeout: 15000
|
||
});
|
||
if (catRes.data.photos?.length > 0) photosList = catRes.data.photos;
|
||
} else {
|
||
console.log('ℹ️ [SerpApi] Kein separater Bad-Ordner auf Google Maps vorhanden.');
|
||
return [];
|
||
}
|
||
|
||
return photosList
|
||
.map(p => p.image || p.thumbnail)
|
||
.filter(Boolean)
|
||
.map(url => url.replace(/=w\d+-h\d+[^=]*/, '=w1200-h800-k-no'))
|
||
.slice(0, 12);
|
||
} catch (error) {
|
||
console.error('❌ [SerpApi] Fehler beim Fotoabruf:', error.message);
|
||
return [];
|
||
}
|
||
}
|
||
|
||
// ==========================================
|
||
// HILFSFUNKTION: HOTELNAME AUS URL EXTRAHIEREN
|
||
// ==========================================
|
||
function extractHotelNameFromUrl(urlString) {
|
||
try {
|
||
const parsed = new URL(urlString);
|
||
const parts = parsed.pathname.split('/').filter(Boolean);
|
||
const lastPart = parts[parts.length - 1] || '';
|
||
const cleanSlug = lastPart.replace(/\.[a-z]{2}\.html$/i, '').replace(/\.html$/i, '');
|
||
if (cleanSlug && cleanSlug.length > 2 && cleanSlug !== 'hotel') {
|
||
return cleanSlug
|
||
.split('-')
|
||
.map(w => w.charAt(0).toUpperCase() + w.slice(1))
|
||
.join(' ');
|
||
}
|
||
} catch (e) {}
|
||
return null;
|
||
}
|
||
|
||
// ==========================================
|
||
// HILFSFUNKTION: URL VON TRACKING & CHALLENGE-PARAMETERN BEREINIGEN
|
||
// ==========================================
|
||
function cleanBookingUrl(rawUrl) {
|
||
try {
|
||
const parsed = new URL(rawUrl);
|
||
// Behalte nur wesentliche Buchungsparameter und entferne challenge tokens (chal_t, force_referer, etc.)
|
||
const allowedParams = ['checkin', 'checkout', 'group_adults', 'no_rooms', 'group_children', 'selected_currency', 'lang'];
|
||
const newParams = new URLSearchParams();
|
||
|
||
for (const param of allowedParams) {
|
||
const val = parsed.searchParams.get(param);
|
||
if (val) newParams.set(param, val);
|
||
}
|
||
|
||
if (!newParams.has('lang')) newParams.set('lang', 'de');
|
||
if (!newParams.has('selected_currency')) newParams.set('selected_currency', 'EUR');
|
||
|
||
parsed.search = newParams.toString();
|
||
parsed.hash = '';
|
||
return parsed.toString();
|
||
} catch (e) {
|
||
return rawUrl;
|
||
}
|
||
}
|
||
|
||
// ==========================================
|
||
// HILFSFUNKTION: SICHERE EVALUATION (RESISTENT GEGEN REDIRECTS)
|
||
// ==========================================
|
||
async function safeEvaluate(page, fn, fallback = null, arg = undefined) {
|
||
try {
|
||
return arg !== undefined ? await page.evaluate(fn, arg) : await page.evaluate(fn);
|
||
} catch (err) {
|
||
if (err.message && (err.message.includes('Execution context was destroyed') || err.message.includes('navigating') || err.message.includes('Target closed'))) {
|
||
await page.waitForLoadState('domcontentloaded').catch(() => {});
|
||
await page.waitForTimeout(2500);
|
||
try {
|
||
return arg !== undefined ? await page.evaluate(fn, arg) : await page.evaluate(fn);
|
||
} catch (innerErr) {
|
||
return fallback;
|
||
}
|
||
}
|
||
return fallback;
|
||
}
|
||
}
|
||
|
||
// ==========================================
|
||
// BULLMQ WORKER: SCRAPING & REVIEWS
|
||
// ==========================================
|
||
const worker = new Worker('scrape-jobs', async (job) => {
|
||
const { url, webhookUrl, appWebhookUrl, hotelId, debug, mode } = job.data;
|
||
const isReviewScan = Boolean((webhookUrl && webhookUrl.startsWith('http')) || mode === 'reviews' || mode === 'full');
|
||
const isDebugMode = Boolean(debug || DEBUG_HTML);
|
||
|
||
console.log(`\n==================================================`);
|
||
console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews + Fotos)' : 'PREIS-SCAN (Schnell)'} für: ${url}`);
|
||
console.log(`==================================================`);
|
||
|
||
let scrapedPrice = null;
|
||
let hasGenius = false;
|
||
let roomSizeM2 = null;
|
||
let hotelDetails = { name: null, rating: null, reviewCount: null, address: null, image: null };
|
||
let domData = { dNights: 1, dPersons: 2, roomSize: null, checkinDate: null, checkoutDate: null };
|
||
let extractedReviews = [];
|
||
let bathroomImages = [];
|
||
let browser = null;
|
||
|
||
try {
|
||
// 1. Browser mit Anti-Bot & Stealth-Optionen starten
|
||
browser = await chromium.launch({
|
||
headless: true,
|
||
args: [
|
||
'--no-sandbox',
|
||
'--disable-setuid-sandbox',
|
||
'--disable-dev-shm-usage',
|
||
'--disable-blink-features=AutomationControlled',
|
||
'--window-size=1920,1080'
|
||
]
|
||
});
|
||
|
||
const context = await browser.newContext({
|
||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
|
||
viewport: { width: 1920, height: 1080 },
|
||
locale: 'de-DE',
|
||
timezoneId: 'Europe/Berlin'
|
||
});
|
||
|
||
// Verhindert Erkennung über navigator.webdriver und emuliert Standard-Browser-Eigenschaften
|
||
await context.addInitScript(() => {
|
||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||
window.chrome = {
|
||
runtime: {},
|
||
loadTimes: function() {},
|
||
csi: function() {},
|
||
app: {}
|
||
};
|
||
Object.defineProperty(navigator, 'plugins', {
|
||
get: () => [1, 2, 3, 4, 5]
|
||
});
|
||
Object.defineProperty(navigator, 'languages', {
|
||
get: () => ['de-DE', 'de', 'en-US', 'en']
|
||
});
|
||
});
|
||
|
||
const page = await context.newPage();
|
||
|
||
let targetUrl = cleanBookingUrl(url);
|
||
|
||
console.log(`[Job ${job.id}] 🌐 Bereinigte Ziel-URL: ${targetUrl}`);
|
||
|
||
await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 45000 });
|
||
await page.waitForLoadState('domcontentloaded').catch(() => {});
|
||
await page.waitForTimeout(2000);
|
||
|
||
// Prüfen, ob AWS WAF Challenge aktiv ist (sicher gegenüber gleichzeitiger Navigation)
|
||
let isWafChallenge = false;
|
||
try {
|
||
isWafChallenge = await page.evaluate(() => {
|
||
return Boolean(
|
||
document.querySelector('#challenge-container') ||
|
||
document.querySelector('script[src*="challenge.js"]') ||
|
||
window.AwsWafIntegration ||
|
||
(document.title === '' && (document.body ? document.body.innerText.includes('verify that you') : false))
|
||
);
|
||
});
|
||
} catch (navErr) {
|
||
await page.waitForLoadState('domcontentloaded').catch(() => {});
|
||
await page.waitForTimeout(2500);
|
||
try {
|
||
isWafChallenge = await page.evaluate(() => {
|
||
return Boolean(
|
||
document.querySelector('#challenge-container') ||
|
||
document.querySelector('script[src*="challenge.js"]') ||
|
||
window.AwsWafIntegration
|
||
);
|
||
});
|
||
} catch (e) {
|
||
isWafChallenge = false;
|
||
}
|
||
}
|
||
|
||
if (isWafChallenge) {
|
||
console.log(`[Job ${job.id}] 🛡️ AWS WAF Challenge erkannt. Warte auf automatische Token-Weiterleitung...`);
|
||
try {
|
||
await page.waitForFunction(() => {
|
||
const hasChallenge = document.querySelector('#challenge-container') || document.querySelector('script[src*="challenge.js"]');
|
||
const hasContent = document.querySelector('h1, h2, #hp_hotel_name, [data-testid="header-title"], script[type="application/ld+json"]');
|
||
return !hasChallenge || Boolean(hasContent);
|
||
}, { timeout: 25000 });
|
||
console.log(`[Job ${job.id}] 🔓 Weiterleitung nach AWS WAF erfolgt!`);
|
||
await page.waitForLoadState('domcontentloaded').catch(() => {});
|
||
await page.waitForTimeout(2000);
|
||
} catch (wafErr) {
|
||
console.log(`[Job ${job.id}] ⚠️ AWS WAF Timeout. Versuche direkten Reload der Basis-URL...`);
|
||
try {
|
||
const parsed = new URL(targetUrl);
|
||
await page.goto(parsed.origin + parsed.pathname, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||
await page.waitForLoadState('domcontentloaded').catch(() => {});
|
||
await page.waitForTimeout(3000);
|
||
} catch (reloadErr) {}
|
||
}
|
||
}
|
||
|
||
// Warte kurz auf wichtige Hotel-Elemente
|
||
try {
|
||
await page.waitForSelector('h1, h2, #hp_hotel_name, [data-testid="header-title"], script[type="application/ld+json"]', { timeout: 8000 });
|
||
} catch (e) {}
|
||
|
||
// Cookie-Banner & störende Overlays für alle Scans sofort schließen
|
||
try {
|
||
const dismissBtns = await page.$$(
|
||
'button[aria-label*="schließen"], button[aria-label*="Schließen"], button[aria-label*="Close"], button[aria-label*="Dismiss"], #onetrust-accept-btn-handler, button:has-text("Akzeptieren"), button:has-text("Alle akzeptieren"), button[data-testid="selection-modal-close-btn"]'
|
||
);
|
||
for (const btn of dismissBtns) {
|
||
await btn.click().catch(() => {});
|
||
await page.waitForTimeout(300);
|
||
}
|
||
} catch (e) {}
|
||
|
||
// HTML-Dump & Screenshot bei Debug-Modus sichern
|
||
if (isDebugMode) {
|
||
try {
|
||
const htmlContent = await page.content();
|
||
await fs.writeFile('src/hotel-debug.html', htmlContent, 'utf-8');
|
||
await page.screenshot({ path: 'src/hotel-debug.png', fullPage: false });
|
||
console.log(`[Job ${job.id}] 📝 Debug-HTML gespeichert unter: scraper-service/src/hotel-debug.html`);
|
||
console.log(`[Job ${job.id}] 📸 Debug-Screenshot gespeichert unter: scraper-service/src/hotel-debug.png`);
|
||
} catch (err) {
|
||
console.error(`[Job ${job.id}] ⚠️ Fehler beim Speichern der Debug-Dateien:`, err.message);
|
||
}
|
||
}
|
||
|
||
// DATUM AUS URL PARSEN
|
||
try {
|
||
const parsedUrl = new URL(url);
|
||
domData.checkinDate = parsedUrl.searchParams.get('checkin');
|
||
domData.checkoutDate = parsedUrl.searchParams.get('checkout');
|
||
} catch (e) {}
|
||
|
||
// 2. STAMMDATEN EXTRAHIEREN (JSON-LD + DOM + META FALLBACKS)
|
||
hotelDetails = await safeEvaluate(page, () => {
|
||
let name = null;
|
||
let rating = null;
|
||
let reviewCount = null;
|
||
let address = null;
|
||
let image = null;
|
||
|
||
// A. JSON-LD durchsuchen (inkl. @graph und verschiedenen Hotel-Typen)
|
||
try {
|
||
const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
|
||
for (const script of scripts) {
|
||
try {
|
||
const raw = JSON.parse(script.textContent || script.innerText || '{}');
|
||
const items = Array.isArray(raw) ? raw : (raw['@graph'] ? raw['@graph'] : [raw]);
|
||
|
||
for (const json of items) {
|
||
const type = (json['@type'] || '').toLowerCase();
|
||
const isAccommodation = ['hotel', 'lodgingbusiness', 'resort', 'apartment', 'hostel', 'motel', 'bedandbreakfast', 'guesthouse', 'inn', 'accommodation'].some(t => type.includes(t));
|
||
|
||
if (isAccommodation || (json.name && (json.address || json.aggregateRating))) {
|
||
if (!name && json.name && typeof json.name === 'string') {
|
||
name = json.name.trim();
|
||
}
|
||
if (!rating && json.aggregateRating?.ratingValue) {
|
||
const r = parseFloat(String(json.aggregateRating.ratingValue).replace(',', '.'));
|
||
if (!isNaN(r)) rating = r;
|
||
}
|
||
if (!reviewCount && json.aggregateRating?.reviewCount) {
|
||
const rc = parseInt(String(json.aggregateRating.reviewCount).replace(/\D/g, ''), 10);
|
||
if (!isNaN(rc)) reviewCount = rc;
|
||
}
|
||
if (!address && json.address) {
|
||
if (typeof json.address === 'string') {
|
||
address = json.address.trim();
|
||
} else if (typeof json.address === 'object') {
|
||
const parts = [
|
||
json.address.streetAddress,
|
||
json.address.postalCode,
|
||
json.address.addressLocality,
|
||
json.address.addressCountry
|
||
].filter(Boolean);
|
||
if (parts.length > 0) address = parts.join(', ').trim();
|
||
}
|
||
}
|
||
if (!image && json.image) {
|
||
if (Array.isArray(json.image) && json.image.length > 0) {
|
||
image = typeof json.image[0] === 'string' ? json.image[0] : json.image[0]?.url;
|
||
} else if (typeof json.image === 'string') {
|
||
image = json.image;
|
||
} else if (json.image?.url) {
|
||
image = json.image.url;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} catch (e) {}
|
||
}
|
||
} catch (e) {}
|
||
|
||
// B. DOM & META FALLBACKS
|
||
|
||
// 1. Name Fallback
|
||
if (!name) {
|
||
const nameSelectors = [
|
||
'h2.pp-header__title',
|
||
'[data-testid="header-title"]',
|
||
'#hp_hotel_name',
|
||
'h2.hp__hotel-name',
|
||
'h2.d2fee87262',
|
||
'.hp__hotel-title h2',
|
||
'h1[data-testid="header-title"]',
|
||
'h1'
|
||
];
|
||
for (const sel of nameSelectors) {
|
||
const el = document.querySelector(sel);
|
||
if (el && el.innerText) {
|
||
const rawText = el.innerText.split('\n')[0].replace(/^(Hotel|Resort|Apartment)\s+/i, '').trim();
|
||
if (rawText && rawText.length > 2 && !rawText.toLowerCase().includes('booking.com')) {
|
||
name = rawText;
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
// Meta Tag Fallback für Name
|
||
if (!name) {
|
||
const ogTitle = document.querySelector('meta[property="og:title"]')?.getAttribute('content');
|
||
if (ogTitle) {
|
||
name = ogTitle.split('–')[0].split('-')[0].split('|')[0].replace(/^(Hotel|Resort|Apartment)\s+/i, '').trim();
|
||
} else if (document.title) {
|
||
name = document.title.split('–')[0].split('-')[0].split('|')[0].replace(/^(Hotel|Resort|Apartment)\s+/i, '').trim();
|
||
}
|
||
}
|
||
}
|
||
|
||
// 2. Adresse Fallback
|
||
if (!address) {
|
||
const addrSelectors = [
|
||
'[data-testid="Address-wrapper"] span',
|
||
'span.hp_address_subtitle',
|
||
'[data-node_tt_id="location_score_tooltip"]',
|
||
'.bui-link--light[data-source="top_link"]',
|
||
'span[data-testid="address-line"]'
|
||
];
|
||
for (const sel of addrSelectors) {
|
||
const el = document.querySelector(sel);
|
||
if (el && el.innerText && el.innerText.trim().length > 5) {
|
||
address = el.innerText.trim().replace(/\s+/g, ' ');
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
// 3. Rating & ReviewCount Fallback
|
||
if (!rating || !reviewCount) {
|
||
try {
|
||
const scoreBlock = document.querySelector('[data-testid="review-score-right-component"], .b5cd09854e, .d8eab2cf7f, [data-testid="review-score-component"]');
|
||
const scoreText = (scoreBlock ? scoreBlock.innerText : (document.body ? document.body.innerText : '')) || '';
|
||
|
||
if (!rating && scoreText) {
|
||
const ratingMatch = scoreText.match(/(?:Bewertet mit|Scored|Punkte|Note|Score)?\s*([0-9][0-9]?[.,][0-9])/i);
|
||
if (ratingMatch) {
|
||
const r = parseFloat(ratingMatch[1].replace(',', '.'));
|
||
if (!isNaN(r) && r <= 10) rating = r;
|
||
}
|
||
}
|
||
|
||
if (!reviewCount && scoreText) {
|
||
const countMatch = scoreText.match(/([0-9][0-9.,]*)\s*(?:Bewertungen|reviews|rezensionen|avis|opiniones|Guest reviews)/i);
|
||
if (countMatch) {
|
||
const rc = parseInt(countMatch[1].replace(/[.,]/g, ''), 10);
|
||
if (!isNaN(rc)) reviewCount = rc;
|
||
}
|
||
}
|
||
} catch (e) {}
|
||
}
|
||
|
||
// 4. Hero-Image Fallback
|
||
if (!image) {
|
||
const ogImage = document.querySelector('meta[property="og:image"]')?.getAttribute('content');
|
||
if (ogImage && ogImage.startsWith('http')) {
|
||
image = ogImage;
|
||
} else {
|
||
const imgSelectors = [
|
||
'[data-testid="hotel-gallery"] img',
|
||
'.gallery_image_round_corners img',
|
||
'#hotel_main_content img',
|
||
'img.bh-photo-grid-thumb'
|
||
];
|
||
for (const sel of imgSelectors) {
|
||
const imgEl = document.querySelector(sel);
|
||
if (imgEl && imgEl.getAttribute('src') && imgEl.getAttribute('src').startsWith('http')) {
|
||
image = imgEl.getAttribute('src');
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
return { name, rating, reviewCount, address, image };
|
||
}, { name: null, rating: null, reviewCount: null, address: null, image: null });
|
||
|
||
// Fallback: Name aus URL-Slug ableiten wenn im HTML nichts gefunden wurde
|
||
if (!hotelDetails.name) {
|
||
hotelDetails.name = extractHotelNameFromUrl(url);
|
||
}
|
||
|
||
// 3. DOM-DATEN (Nächte, Personen, Zimmergröße)
|
||
const extraDom = await safeEvaluate(page, () => {
|
||
try {
|
||
const bodyText = document.body ? (document.body.innerText || '') : '';
|
||
const nMatch = bodyText.match(/(\d+)\s*Nächte?/i);
|
||
const pMatch = bodyText.match(/(\d+)\s*(?:Erwachsene|Erwachsener|Gäste|Personen)/i);
|
||
const rmMatch = bodyText.match(/(\d+)\s*m²/i);
|
||
return {
|
||
dNights: nMatch ? parseInt(nMatch[1], 10) : 1,
|
||
dPersons: pMatch ? parseInt(pMatch[1], 10) : 2,
|
||
roomSize: rmMatch ? parseInt(rmMatch[1], 10) : null
|
||
};
|
||
} catch (e) {
|
||
return { dNights: 1, dPersons: 2, roomSize: null };
|
||
}
|
||
}, { dNights: 1, dPersons: 2, roomSize: null });
|
||
|
||
domData.dNights = extraDom.dNights;
|
||
domData.dPersons = extraDom.dPersons;
|
||
roomSizeM2 = extraDom.roomSize;
|
||
|
||
// PREIS AUSLESEN
|
||
scrapedPrice = await safeEvaluate(page, () => {
|
||
try {
|
||
const selectors = [
|
||
'[data-testid="price-and-discounted-price"]',
|
||
'.totalPrice .bui-price-display__value',
|
||
'.hprt-price-price',
|
||
'.prco-valign-middle-helper',
|
||
'[data-testid="price-display"]',
|
||
'span.prco-inline-block-maker-helper',
|
||
'.bui-price-display__value'
|
||
];
|
||
for (const sel of selectors) {
|
||
const el = document.querySelector(sel);
|
||
if (el && el.innerText && /\d/.test(el.innerText)) return el.innerText.trim();
|
||
}
|
||
|
||
const bodyText = document.body ? (document.body.innerText || '') : '';
|
||
const priceMatch = bodyText.match(/(?:€|EUR)\s*([0-9][0-9.,]*)|([0-9][0-9.,]*)\s*(?:€|EUR)/i);
|
||
if (priceMatch) {
|
||
return (priceMatch[1] || priceMatch[2] || '').trim();
|
||
}
|
||
} catch (e) {}
|
||
return null;
|
||
}, null);
|
||
|
||
// GENIUS BADGE PRÜFUNG
|
||
hasGenius = await safeEvaluate(page, () => {
|
||
try {
|
||
const geniusSelectors = [
|
||
'.m-badge__genius',
|
||
'svg.genius-badge',
|
||
'.-genius-new_identity-genius_badge',
|
||
'.c-deals-container__genius-badge-fix',
|
||
'[aria-label*="Genius"]',
|
||
'[data-testid="deal-badge"]'
|
||
];
|
||
return geniusSelectors.some(sel => Boolean(document.querySelector(sel)));
|
||
} catch (e) {
|
||
return false;
|
||
}
|
||
}, false);
|
||
|
||
console.log(`[Job ${job.id}] 📋 Extrahierte Stammdaten:`, {
|
||
name: hotelDetails.name,
|
||
rating: hotelDetails.rating,
|
||
reviewCount: hotelDetails.reviewCount,
|
||
address: hotelDetails.address,
|
||
image: hotelDetails.image ? (hotelDetails.image.substring(0, 60) + '...') : null,
|
||
price: scrapedPrice,
|
||
hasGenius,
|
||
nights: domData.dNights,
|
||
persons: domData.dPersons,
|
||
roomSizeM2
|
||
});
|
||
|
||
if (hasGenius) {
|
||
console.log(`[Job ${job.id}] 💙 Genius-Badge erkannt (-10% Rabatt gilt)!`);
|
||
}
|
||
|
||
// 4. REVIEWS EXTRAHIEREN (STICKY-HEADER AKTIVIERUNG & ECHTER MAUSKLICK)
|
||
if (isReviewScan) {
|
||
console.log(`[Job ${job.id}] 🚀 Starte gezielte Review-Extraction für "${hotelDetails.name || 'Hotel'}"...`);
|
||
|
||
// B. Herunterscrollen um Lazy-Components zu hydrieren und Review-Dialog zu öffnen
|
||
console.log(`[Job ${job.id}] 📜 Scrolle herunter, um verzögert geladene Review-Komponenten zu aktivieren...`);
|
||
await page.evaluate(() => window.scrollTo(0, 4000));
|
||
await page.waitForTimeout(2500);
|
||
|
||
// C. Klick auf Review-Trigger ausführen
|
||
console.log(`[Job ${job.id}] 👉 Öffne Review-Dialog über 'Alle Bewertungen lesen' / 'Gästebewertungen'...`);
|
||
await page.evaluate(() => {
|
||
const btn = document.querySelector('button[data-testid="fr-read-all-reviews"]') ||
|
||
document.querySelector('button[data-testid="review-score-read-all-actionable"]') ||
|
||
document.getElementById('reviews-tab-trigger') ||
|
||
document.querySelector('[data-testid="Property-Header-Nav-Tab-Trigger-reviews"]');
|
||
if (btn) {
|
||
btn.scrollIntoView({ behavior: 'instant', block: 'center' });
|
||
btn.click();
|
||
btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
|
||
return true;
|
||
}
|
||
return false;
|
||
}).catch(() => false);
|
||
|
||
// 4 Sekunden warten bis das Dialog-Panel mit den 1.000+ Bewertungen vollständig geöffnet ist
|
||
await page.waitForTimeout(4000);
|
||
|
||
// Screenshot zur visuellen Kontrolle aktualisieren
|
||
try {
|
||
await page.screenshot({ path: 'src/review-debug.png', fullPage: false });
|
||
console.log(`[Job ${job.id}] 📸 Screenshot aktualisiert unter: scraper-service/src/review-debug.png`);
|
||
} catch (e) {}
|
||
|
||
// D. Sortierung auf "Neueste zuerst" (NEWEST_FIRST) umstellen
|
||
try {
|
||
const sorterLocator = page.locator('#reviewListSorters, [data-testid="reviews-sorter-component"], select[name="reviewListSorters"]').first();
|
||
if (await sorterLocator.count() > 0) {
|
||
console.log(`[Job ${job.id}] 🔄 Stelle Sortierung auf 'NEWEST_FIRST' (Neueste zuerst) um...`);
|
||
await sorterLocator.selectOption('NEWEST_FIRST');
|
||
await page.waitForTimeout(3000); // Warten bis Booking die Liste per AJAX neu lädt
|
||
}
|
||
} catch (e) {
|
||
console.log(`[Job ${job.id}] ⚠️ Sortierung konnte nicht angepasst werden:`, e.message);
|
||
}
|
||
|
||
let pageCount = 0;
|
||
const seenKeys = new Set();
|
||
|
||
while (pageCount < 30) {
|
||
pageCount++;
|
||
console.log(`[Job ${job.id}] 📄 Scanne Bewertungsseite ${pageCount} (Bisher gesammelt: ${extractedReviews.length})...`);
|
||
|
||
// Extrahiere alle Review-Karten aus dem Dialog / DOM
|
||
const pageReviews = await page.evaluate(() => {
|
||
const cards = Array.from(document.querySelectorAll(
|
||
'#reviewCardsSection [data-testid="review-card"], div[role="dialog"] [data-testid="review-card"], [data-testid="review-card"]'
|
||
));
|
||
const items = [];
|
||
const currentYear = new Date().getFullYear();
|
||
|
||
for (const card of cards) {
|
||
const titleEl = card.querySelector('[data-testid="review-title"], h4');
|
||
const posEl = card.querySelector('[data-testid="review-positive-text"]');
|
||
const negEl = card.querySelector('[data-testid="review-negative-text"]');
|
||
const dateEl = card.querySelector('[data-testid="review-date"]');
|
||
const flagImg = card.querySelector('[data-testid="review-avatar"] .fff1944c52 img[alt], [data-testid="review-country"] img[alt], img.b8d1620349');
|
||
const countrySpan = card.querySelector('[data-testid="review-avatar"] .fff1944c52 span:not(.cec4c9cfef), [data-testid="review-country"]');
|
||
|
||
const title = titleEl ? titleEl.innerText.trim() : '';
|
||
const positive = posEl ? posEl.innerText.trim().replace(/^„|“$/g, '').trim() : '';
|
||
const negative = negEl ? negEl.innerText.trim().replace(/^„|“$/g, '').trim() : '';
|
||
const dateText = dateEl ? dateEl.innerText.trim() : 'Aktuell';
|
||
const country = (flagImg ? flagImg.getAttribute('alt') : (countrySpan ? countrySpan.innerText : '')).trim();
|
||
|
||
const cardText = card.innerText || '';
|
||
const yearMatch = cardText.match(/\b(20\d{2})\b/);
|
||
if (yearMatch) {
|
||
const reviewYear = parseInt(yearMatch[1], 10);
|
||
if (currentYear - reviewYear > 1) {
|
||
continue;
|
||
}
|
||
}
|
||
|
||
let fallbackText = '';
|
||
if (!positive && !negative) {
|
||
fallbackText = cardText.replace(title, '').substring(0, 300).trim();
|
||
}
|
||
|
||
if (title || positive || negative || fallbackText) {
|
||
items.push({ title, positive: positive || fallbackText, negative, date: dateText, country });
|
||
}
|
||
}
|
||
return items;
|
||
}).catch(err => {
|
||
console.error(`[Job ${job.id}] Evaluate error on page ${pageCount}:`, err.message);
|
||
return [];
|
||
});
|
||
|
||
console.log(`[Job ${job.id}] 🔍 Echte Gästekarten auf Seite ${pageCount} gefunden: ${pageReviews.length}`);
|
||
|
||
for (const rev of pageReviews) {
|
||
if (rev.country) {
|
||
const isWestern = WESTERN_COUNTRIES.some(c => rev.country.toLowerCase().includes(c.toLowerCase()));
|
||
if (!isWestern) {
|
||
console.log(`[Job ${job.id}] └─ ❌ Land nicht in Liste: "${rev.country}"`);
|
||
continue;
|
||
}
|
||
}
|
||
const key = `${rev.title}-${rev.positive}-${rev.date}`;
|
||
if (!seenKeys.has(key) && extractedReviews.length < 80) {
|
||
seenKeys.add(key);
|
||
extractedReviews.push(rev);
|
||
console.log(`[Job ${job.id}] └─ ✅ Akzeptiert: "${rev.title || rev.positive.substring(0, 30)}" (${rev.country || 'International'})`);
|
||
}
|
||
}
|
||
|
||
if (extractedReviews.length >= 80) break;
|
||
|
||
// Weiterblättern im Modal
|
||
const nextResult = await page.evaluate(() => {
|
||
const nextBtn = document.querySelector('div[role="dialog"] button[aria-label="Nächste Seite"], button[aria-label="Nächste Seite"], button[aria-label*="Nächste Seite"]');
|
||
if (nextBtn && !nextBtn.disabled && !nextBtn.hasAttribute('disabled')) {
|
||
nextBtn.scrollIntoView({ behavior: 'instant', block: 'center' });
|
||
nextBtn.click();
|
||
nextBtn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
|
||
return true;
|
||
}
|
||
return false;
|
||
}).catch(() => false);
|
||
|
||
if (nextResult) {
|
||
await page.waitForTimeout(3000);
|
||
} else {
|
||
break;
|
||
}
|
||
}
|
||
|
||
console.log(`[Job ${job.id}] ✅ Insgesamt ${extractedReviews.length} passende Kommentare extrahiert.`);
|
||
}
|
||
|
||
} catch (error) {
|
||
console.error(`[Job ${job.id}] ❌ Fehler beim Scraping:`, error.message);
|
||
} finally {
|
||
if (browser) await browser.close();
|
||
|
||
// 5. GOOGLE MAPS FOTOS NUR BEI KI-REVIEW SCAN ABRUFEN
|
||
if (isReviewScan && hotelDetails.name) {
|
||
try {
|
||
bathroomImages = await getGoogleBathroomPhotosViaApi(hotelDetails.name, hotelDetails.address);
|
||
} catch (err) {
|
||
console.error('Foto-Abruf fehlgeschlagen:', err.message);
|
||
bathroomImages = [];
|
||
}
|
||
}
|
||
|
||
// 6. PREISHISTORIE + STAMMDATEN AN NEXT.JS WEBHOOK SENDEN
|
||
if (appWebhookUrl) {
|
||
try {
|
||
await axios.post(appWebhookUrl, {
|
||
price: scrapedPrice,
|
||
hasGenius: hasGenius,
|
||
roomSizeM2,
|
||
hotelName: hotelDetails.name,
|
||
rating: hotelDetails.rating,
|
||
reviewCount: hotelDetails.reviewCount,
|
||
address: hotelDetails.address,
|
||
image: hotelDetails.image,
|
||
checkinDate: domData.checkinDate,
|
||
checkoutDate: domData.checkoutDate,
|
||
nights: domData.dNights || 1,
|
||
persons: domData.dPersons || 2,
|
||
bathroomImages: Array.isArray(bathroomImages) && bathroomImages.length > 0 ? bathroomImages : undefined
|
||
});
|
||
console.log(`[Job ${job.id}] ✅ Preishistorie & Stammdaten in Postgres gespeichert!`);
|
||
} catch (e) {
|
||
console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie-Webhook:`, e.message);
|
||
}
|
||
}
|
||
|
||
// 7. REVIEWS AN N8N WEBHOOK SENDEN
|
||
if (isReviewScan && extractedReviews.length > 0) {
|
||
try {
|
||
await axios.post(webhookUrl, {
|
||
jobId: job.id,
|
||
hotelId,
|
||
hotelName: hotelDetails.name,
|
||
reviews: extractedReviews,
|
||
timestamp: new Date().toISOString()
|
||
});
|
||
console.log(`[Job ${job.id}] 🚀 ${extractedReviews.length} Reviews an n8n gesendet!`);
|
||
} catch (e) {
|
||
console.error(`[Job ${job.id}] ❌ Fehler beim Senden an n8n:`, e.message);
|
||
}
|
||
}
|
||
}
|
||
}, { ...REDIS_CONFIG, concurrency: 2 });
|
||
|
||
console.log('🚀 Scraper Worker bereit...'); |