gitea/booking--scraper-service-
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(scraper): perfect review dialog opening, pagination, and multi-page review extraction

Browse Source
This commit is contained in:
gitea
2026-08-21 21:40:16 +00:00
parent 8cf101dee5
commit 5882540cf8
1 changed files with 94 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

276
src/index.js
View File

@@ -23,7 +23,10 @@ const WESTERN_COUNTRIES = [
'United Kingdom', 'UK', 'Niederlande', 'Netherlands', 'Frankreich', 'France', 'United Kingdom', 'UK', 'Niederlande', 'Netherlands', 'Frankreich', 'France',
'Spanien', 'Spain', 'Italien', 'Italy', 'Kanada', 'Canada', 'Australien', 'Australia', 'Spanien', 'Spain', 'Italien', 'Italy', 'Kanada', 'Canada', 'Australien', 'Australia',
'Belgien', 'Belgium', 'Schweden', 'Sweden', 'Norwegen', 'Norway', 'Dänemark', 'Denmark', 'Belgien', 'Belgium', 'Schweden', 'Sweden', 'Norwegen', 'Norway', 'Dänemark', 'Denmark',
'Finnland', 'Finland', 'Irland', 'Ireland', 'Luxemburg', 'Luxembourg', 'Indien', 'India' 'Finnland', 'Finland', 'Irland', 'Ireland', 'Luxemburg', 'Luxembourg', 'Indien', 'India',
'Singapur', 'Singapore', 'Malaysia', 'Sri Lanka', 'Neuseeland', 'New Zealand',
'Südafrika', 'South Africa', 'Vereinigte Arabische Emirate', 'UAE', 'Dubai',
'Polen', 'Poland', 'Tschechien', 'Czech', 'Japan', 'Korea'
]; ];
// ========================================== // ==========================================
@@ -163,8 +166,8 @@ async function safeEvaluate(page, fn, fallback = null, arg = undefined) {
// BULLMQ WORKER: SCRAPING & REVIEWS // BULLMQ WORKER: SCRAPING & REVIEWS
// ========================================== // ==========================================
const worker = new Worker('scrape-jobs', async (job) => { const worker = new Worker('scrape-jobs', async (job) => {
const { url, webhookUrl, appWebhookUrl, hotelId, debug } = job.data; const { url, webhookUrl, appWebhookUrl, hotelId, debug, mode } = job.data;
const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http')); const isReviewScan = Boolean((webhookUrl && webhookUrl.startsWith('http')) || mode === 'reviews' || mode === 'full');
const isDebugMode = Boolean(debug || DEBUG_HTML); const isDebugMode = Boolean(debug || DEBUG_HTML);
console.log(`\n==================================================`); console.log(`\n==================================================`);
@@ -187,29 +190,21 @@ const worker = new Worker('scrape-jobs', async (job) => {
args: [ args: [
'--no-sandbox', '--no-sandbox',
'--disable-setuid-sandbox', '--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-blink-features=AutomationControlled', '--disable-blink-features=AutomationControlled',
'--disable-infobars' '--window-size=1920,1080'
] ]
}); });
const context = await browser.newContext({ const context = await browser.newContext({
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
viewport: { width: 1366, height: 900 }, viewport: { width: 1920, height: 1080 },
locale: 'de-DE', locale: 'de-DE',
timezoneId: 'Europe/Berlin', timezoneId: 'Europe/Berlin'
extraHTTPHeaders: {
'Accept-Language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
'Upgrade-Insecure-Requests': '1',
'Sec-Ch-Ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"',
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Ch-Ua-Platform': '"Windows"'
}
}); });
const page = await context.newPage();
// Verhindert Erkennung über navigator.webdriver und emuliert Standard-Browser-Eigenschaften // Verhindert Erkennung über navigator.webdriver und emuliert Standard-Browser-Eigenschaften
await page.addInitScript(() => { await context.addInitScript(() => {
Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
window.chrome = { window.chrome = {
runtime: {}, runtime: {},
@@ -225,6 +220,8 @@ const worker = new Worker('scrape-jobs', async (job) => {
}); });
}); });
const page = await context.newPage();
let targetUrl = cleanBookingUrl(url); let targetUrl = cleanBookingUrl(url);
console.log(`[Job ${job.id}] 🌐 Bereinigte Ziel-URL: ${targetUrl}`); console.log(`[Job ${job.id}] 🌐 Bereinigte Ziel-URL: ${targetUrl}`);
@@ -289,20 +286,12 @@ const worker = new Worker('scrape-jobs', async (job) => {
// Cookie-Banner & störende Overlays für alle Scans sofort schließen // Cookie-Banner & störende Overlays für alle Scans sofort schließen
try { try {
const dismissSelectors = [ const dismissBtns = await page.$$(
'#onetrust-accept-btn-handler', 'button[aria-label*="schließen"], button[aria-label*="Schließen"], button[aria-label*="Close"], button[aria-label*="Dismiss"], #onetrust-accept-btn-handler, button:has-text("Akzeptieren"), button:has-text("Alle akzeptieren"), button[data-testid="selection-modal-close-btn"]'
'button#onetrust-reject-all-handler', );
'button[data-testid="selection-modal-close-btn"]', for (const btn of dismissBtns) {
'button[aria-label="Schließen"]', await btn.click().catch(() => {});
'button[aria-label="Close"]', await page.waitForTimeout(300);
'button.modal-mask-closeBtn'
];
for (const btnSel of dismissSelectors) {
const btn = await page.$(btnSel);
if (btn) {
await btn.click().catch(() => {});
await page.waitForTimeout(300);
}
} }
} catch (e) {} } catch (e) {}
@@ -579,67 +568,35 @@ const worker = new Worker('scrape-jobs', async (job) => {
if (isReviewScan) { if (isReviewScan) {
console.log(`[Job ${job.id}] 🚀 Starte gezielte Review-Extraction für "${hotelDetails.name || 'Hotel'}"...`); console.log(`[Job ${job.id}] 🚀 Starte gezielte Review-Extraction für "${hotelDetails.name || 'Hotel'}"...`);
// A. Cookie-Banner und störende Overlays schließen
try {
const dismissSelectors = [
'#onetrust-accept-btn-handler',
'button#onetrust-reject-all-handler',
'button[data-testid="selection-modal-close-btn"]',
'button[aria-label="Schließen"]',
'button[aria-label="Close"]'
];
for (const btnSel of dismissSelectors) {
const btn = await page.$(btnSel);
if (btn) {
await btn.click().catch(() => {});
await page.waitForTimeout(400);
}
}
} catch (e) {}
// B. Herunterscrollen um Lazy-Components zu hydrieren und Review-Dialog zu öffnen // B. Herunterscrollen um Lazy-Components zu hydrieren und Review-Dialog zu öffnen
console.log(`[Job ${job.id}] 📜 Scrolle herunter, um verzögert geladene Review-Komponenten zu aktivieren...`); console.log(`[Job ${job.id}] 📜 Scrolle herunter, um verzögert geladene Review-Komponenten zu aktivieren...`);
await page.evaluate(() => window.scrollTo(0, 4000)); await page.evaluate(() => window.scrollTo(0, 4000));
await page.waitForTimeout(2000); await page.waitForTimeout(2500);
// C. Klick auf Review-Trigger ausführen // C. Klick auf Review-Trigger ausführen
let isPanelOpen = false; console.log(`[Job ${job.id}] 👉 Öffne Review-Dialog über 'Alle Bewertungen lesen' / 'Gästebewertungen'...`);
try { await page.evaluate(() => {
console.log(`[Job ${job.id}] 👉 Öffne Review-Dialog über 'Alle Bewertungen lesen' / 'Gästebewertungen'...`); const btn = document.querySelector('button[data-testid="fr-read-all-reviews"]') ||
isPanelOpen = await page.evaluate(() => { document.querySelector('button[data-testid="review-score-read-all-actionable"]') ||
const btn = document.querySelector('button[data-testid="fr-read-all-reviews"]') || document.getElementById('reviews-tab-trigger') ||
document.querySelector('button[data-testid="review-score-read-all-actionable"]') || document.querySelector('[data-testid="Property-Header-Nav-Tab-Trigger-reviews"]');
document.getElementById('reviews-tab-trigger') || if (btn) {
document.querySelector('[data-testid="Property-Header-Nav-Tab-Trigger-reviews"]'); btn.scrollIntoView({ behavior: 'instant', block: 'center' });
if (btn) { btn.click();
btn.scrollIntoView({ behavior: 'instant', block: 'center' }); btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
btn.click(); return true;
btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
return true;
}
return false;
}).catch(() => false);
// Warten bis das Dialog-Panel mit den 1.000+ Bewertungen gerendert ist
try {
await page.waitForSelector(
'div[role="dialog"] [data-testid="review-card"], #reviewCardsSection [data-testid="review-card"], [data-testid="review-card"]',
{ timeout: 15000 }
);
console.log(`[Job ${job.id}] 🌟 Review-Dialog ('role="dialog"') mit allen Bewertungen erfolgreich geladen!`);
} catch (waitErr) {
console.log(`[Job ${job.id}] ⚠️ Dialog-Karten nicht sofort erkannt, fahre mit vorhandenem DOM fort...`);
} }
return false;
}).catch(() => false);
// Screenshot zur visuellen Kontrolle aktualisieren // 4 Sekunden warten bis das Dialog-Panel mit den 1.000+ Bewertungen vollständig geöffnet ist
try { await page.waitForTimeout(4000);
await page.screenshot({ path: 'src/review-debug.png', fullPage: false });
console.log(`[Job ${job.id}] 📸 Screenshot aktualisiert unter: scraper-service/src/review-debug.png`);
} catch (e) {}
} catch (e) { // Screenshot zur visuellen Kontrolle aktualisieren
console.log(`[Job ${job.id}] ⚠️ Fehler beim Klick auf Review-Trigger:`, e.message); try {
} await page.screenshot({ path: 'src/review-debug.png', fullPage: false });
console.log(`[Job ${job.id}] 📸 Screenshot aktualisiert unter: scraper-service/src/review-debug.png`);
} catch (e) {}
// D. Sortierung auf "Neueste zuerst" (NEWEST_FIRST) umstellen // D. Sortierung auf "Neueste zuerst" (NEWEST_FIRST) umstellen
try { try {
@@ -660,84 +617,36 @@ const worker = new Worker('scrape-jobs', async (job) => {
pageCount++; pageCount++;
console.log(`[Job ${job.id}] 📄 Scanne Bewertungsseite ${pageCount} (Bisher gesammelt: ${extractedReviews.length})...`); console.log(`[Job ${job.id}] 📄 Scanne Bewertungsseite ${pageCount} (Bisher gesammelt: ${extractedReviews.length})...`);
const evalResult = await safeEvaluate(page, ({ westernCountries }) => { // Extrahiere alle Review-Karten aus dem Dialog / DOM
let cards = Array.from(document.querySelectorAll([ const pageReviews = await page.evaluate(() => {
'#reviewCardsSection [data-testid="review-card"]', const cards = Array.from(document.querySelectorAll(
'div[role="dialog"] [data-testid="review-card"]', '#reviewCardsSection [data-testid="review-card"], div[role="dialog"] [data-testid="review-card"], [data-testid="review-card"]'
'[data-testid="review-card"]', ));
'[data-testid="review-cards"] > li', const items = [];
'[data-testid="review-cards"] > div',
'div[data-testid*="review-card"]',
'li[data-testid*="review-card"]',
'li.review_list_new_item_block',
'div.c-review-block__item',
'div[itemprop="review"]',
'li.review_item',
'div.c-review',
'div.review_item',
'.c-review-block',
'li.c-review-block'
].join(',')));
if (cards.length === 0) {
const container = document.querySelector('#hp-reviews-sliding, #reviewCardsSection, [data-testid="review-cards"], div[role="dialog"]');
if (container) {
cards = Array.from(container.querySelectorAll('li, div.c-review-block, div.c-review')).filter(el => (el.innerText || '').length > 30);
}
}
// Filterboxen und Scorecard-Zusammenfassungen explizit ausschließen
cards = cards.filter(el => {
const text = el.innerText || '';
return text.length > 20 &&
!text.includes('Wählen Sie Themen') &&
!text.includes('Kategorien:') &&
!el.classList.contains('review_score_breakdown');
});
const results = [];
const debugLogs = [];
const currentYear = 2026; const currentYear = 2026;
for (let idx = 0; idx < cards.length; idx++) { for (const card of cards) {
const card = cards[idx]; const titleEl = card.querySelector('[data-testid="review-title"], h4');
const cardText = card.innerText || ''; const posEl = card.querySelector('[data-testid="review-positive-text"]');
if (!cardText.trim()) continue; const negEl = card.querySelector('[data-testid="review-negative-text"]');
const dateEl = card.querySelector('[data-testid="review-date"]');
// 1. Herkunftsland des Gastes ermitteln
const flagImg = card.querySelector('[data-testid="review-avatar"] .fff1944c52 img[alt], [data-testid="review-country"] img[alt], img.b8d1620349'); const flagImg = card.querySelector('[data-testid="review-avatar"] .fff1944c52 img[alt], [data-testid="review-country"] img[alt], img.b8d1620349');
const countrySpan = card.querySelector('[data-testid="review-avatar"] .fff1944c52 span:not(.cec4c9cfef), [data-testid="review-country"]'); const countrySpan = card.querySelector('[data-testid="review-avatar"] .fff1944c52 span:not(.cec4c9cfef), [data-testid="review-country"]');
const guestCountryText = (flagImg ? flagImg.getAttribute('alt') : (countrySpan ? countrySpan.innerText : '')).trim();
if (guestCountryText && guestCountryText.length > 1) {
const isWestern = westernCountries.some(c => guestCountryText.toLowerCase().includes(c.toLowerCase()));
if (!isWestern) {
const preview = guestCountryText.substring(0, 50).replace(/\n/g, ' ');
debugLogs.push(`Karte #${idx + 1}: ❌ Land nicht in Liste ("${preview}")`);
continue;
}
}
// 2. Datum / Jahr prüfen (letzte 2 Jahre)
const yearMatch = cardText.match(/\b(20\d{2})\b/);
if (yearMatch) {
const reviewYear = parseInt(yearMatch[1], 10);
if (currentYear - reviewYear > 2) {
debugLogs.push(`Karte #${idx + 1}: ❌ Zu alt (${reviewYear})`);
continue;
}
}
// 3. Texte extrahieren
const titleEl = card.querySelector('[data-testid="review-title"], .c-review__title, h4, .review_title');
const posEl = card.querySelector('[data-testid="review-positive-text"], .c-review__row--positive, .review_pos');
const negEl = card.querySelector('[data-testid="review-negative-text"], .c-review__row--negative, .review_neg');
const dateEl = card.querySelector('[data-testid="review-date"], .c-review__date, .review_item_date');
const title = titleEl ? titleEl.innerText.trim() : ''; const title = titleEl ? titleEl.innerText.trim() : '';
const positive = posEl ? posEl.innerText.trim().replace(/^„|“$/g, '').trim() : ''; const positive = posEl ? posEl.innerText.trim().replace(/^„|“$/g, '').trim() : '';
const negative = negEl ? negEl.innerText.trim().replace(/^„|“$/g, '').trim() : ''; const negative = negEl ? negEl.innerText.trim().replace(/^„|“$/g, '').trim() : '';
const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell'); const dateText = dateEl ? dateEl.innerText.trim() : 'Aktuell';
const country = (flagImg ? flagImg.getAttribute('alt') : (countrySpan ? countrySpan.innerText : '')).trim();
const cardText = card.innerText || '';
const yearMatch = cardText.match(/\b(20\d{2})\b/);
if (yearMatch) {
const reviewYear = parseInt(yearMatch[1], 10);
if (currentYear - reviewYear > 2) {
continue;
}
}
let fallbackText = ''; let fallbackText = '';
if (!positive && !negative) { if (!positive && !negative) {
@@ -745,47 +654,50 @@ const worker = new Worker('scrape-jobs', async (job) => {
} }
if (title || positive || negative || fallbackText) { if (title || positive || negative || fallbackText) {
results.push({ title, positive: positive || fallbackText, negative, date: dateText }); items.push({ title, positive: positive || fallbackText, negative, date: dateText, country });
debugLogs.push(`Karte #${idx + 1}: ✅ Akzeptiert ("${title || (positive ? positive.substring(0, 25) : 'Review')}")`);
} }
} }
return items;
}).catch(err => {
console.error(`[Job ${job.id}] Evaluate error on page ${pageCount}:`, err.message);
return [];
});
return { console.log(`[Job ${job.id}] 🔍 Echte Gästekarten auf Seite ${pageCount} gefunden: ${pageReviews.length}`);
totalCards: cards.length,
results,
debugLogs: debugLogs.slice(0, 8)
};
}, { totalCards: 0, results: [], debugLogs: [] }, { westernCountries: WESTERN_COUNTRIES });
console.log(`[Job ${job.id}] 🔍 Echte Gästekarten gefunden: ${evalResult.totalCards}`); for (const rev of pageReviews) {
if (evalResult.debugLogs) { if (rev.country) {
evalResult.debugLogs.forEach(log => console.log(`[Job ${job.id}] └─ ${log}`)); const isWestern = WESTERN_COUNTRIES.some(c => rev.country.toLowerCase().includes(c.toLowerCase()));
} if (!isWestern) {
console.log(`[Job ${job.id}] └─ ❌ Land nicht in Liste: "${rev.country}"`);
// Deduplizierung continue;
for (const rev of evalResult.results) { }
}
const key = `${rev.title}-${rev.positive}-${rev.date}`; const key = `${rev.title}-${rev.positive}-${rev.date}`;
if (!seenKeys.has(key) && extractedReviews.length < 80) { if (!seenKeys.has(key) && extractedReviews.length < 80) {
seenKeys.add(key); seenKeys.add(key);
extractedReviews.push(rev); extractedReviews.push(rev);
console.log(`[Job ${job.id}] └─ ✅ Akzeptiert: "${rev.title || rev.positive.substring(0, 30)}" (${rev.country || 'International'})`);
} }
} }
if (extractedReviews.length >= 80) break; if (extractedReviews.length >= 80) break;
// Weiterblättern im Modal oder der Hauptseite // Weiterblättern im Modal
try { const nextResult = await page.evaluate(() => {
const nextBtn = await page.$( const nextBtn = document.querySelector('div[role="dialog"] button[aria-label="Nächste Seite"], button[aria-label="Nächste Seite"], button[aria-label*="Nächste Seite"]');
'div[role="dialog"] button[aria-label="Nächste Seite"], button[aria-label="Nächste Seite"], button[aria-label*="Nächste Seite"], #hp-reviews-sliding button[aria-label*="Nächste"], button[aria-label*="Next"], #reviewCardsSection button:has(svg), a.paging-next, li.bui-pagination__next-arrow a' if (nextBtn && !nextBtn.disabled && !nextBtn.hasAttribute('disabled')) {
); nextBtn.scrollIntoView({ behavior: 'instant', block: 'center' });
if (nextBtn) { nextBtn.click();
const isDisabled = await nextBtn.getAttribute('disabled'); nextBtn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
if (isDisabled === null) { return true;
await nextBtn.click({ force: true }); }
await page.waitForTimeout(2500); return false;
} else break; }).catch(() => false);
} else break;
} catch (e) { if (nextResult) {
await page.waitForTimeout(3000);
} else {
break; break;
} }
} }