gitea/booking--scraper-service-
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(scraper): enable reliable multi-page review dialog extraction with lazy-scroll and exact country parsing

Browse Source
This commit is contained in:
gitea
2026-08-21 21:11:06 +00:00
parent 28d52be2c3
commit 8cf101dee5
4 changed files with 134 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -11,3 +11,4 @@ src/*.png
debug/
.DS_Store

21
AGENTS.md Normal file
View File

@@ -0,0 +1,21 @@
# Engineering & Scraping Guidelines
## 1. Git Commit & Repository Hygiene
- **Conventional Commits:** Write clear, concise commit messages prefixed with standard types (e.g., `feat:`, `fix:`, `refactor:`, `docs:`).
- **Exclude Transient Artifacts:** Never stage or commit generated debug files (e.g., `src/*.html`, `src/*.png`, large logs) to version control. Always maintain `.gitignore` to keep git packs lean and prevent remote unpacker errors on Gitea/GitHub.
## 2. Web Scraping & DOM Inspection Strategy
- **Rendered DOM Dumping for Debugging:** When debugging extraction failures on dynamic or anti-bot protected pages:
- Capture the post-render HTML via `await page.content()` and save it to `src/hotel-debug.html`.
- Capture a synchronized screenshot (`src/hotel-debug.png`).
- Inspect the raw rendered DOM to verify if the page was served as standard SSR, an A/B test SPA variant, or intercepted by a challenge (e.g. AWS WAF).
- **Multi-Tier Extraction Hierarchy:** Always implement fallbacks in this order:
1. **Deep JSON-LD Parsing:** Traverse all `<script type="application/ld+json">` tags, including `@graph` arrays and varied accommodation schemas (`Hotel`, `Resort`, `Apartment`, `Hostel`, `LodgingBusiness`, etc.).
2. **DOM Selectors:** Fallback to standard test IDs, classes, and element hierarchy (`data-testid`, header titles, score components).
3. **Meta Tags:** Fallback to OpenGraph and Twitter meta tags (`og:title`, `og:image`, `og:description`).
4. **URL Slug Extraction:** As an ultimate safety net, derive clean human-readable names from the URL path if the HTML body was obstructed.
- **Anti-Bot & WAF Handling:**
- Sanitize input URLs by stripping transient tokens (`chal_t`, `force_referer`, session IDs).
- Emulate full browser properties (`window.chrome`, `navigator.plugins`, `navigator.languages`).
- Actively detect WAF challenge interstitial pages and wait for client-side JavaScript token computation and automatic redirection before evaluating the DOM.

196
src/index.js
View File

@@ -139,6 +139,26 @@ function cleanBookingUrl(rawUrl) {
}
}
// ==========================================
// HILFSFUNKTION: SICHERE EVALUATION (RESISTENT GEGEN REDIRECTS)
// ==========================================
async function safeEvaluate(page, fn, fallback = null, arg = undefined) {
try {
return arg !== undefined ? await page.evaluate(fn, arg) : await page.evaluate(fn);
} catch (err) {
if (err.message && (err.message.includes('Execution context was destroyed') || err.message.includes('navigating') || err.message.includes('Target closed'))) {
await page.waitForLoadState('domcontentloaded').catch(() => {});
await page.waitForTimeout(2500);
try {
return arg !== undefined ? await page.evaluate(fn, arg) : await page.evaluate(fn);
} catch (innerErr) {
return fallback;
}
}
return fallback;
}
}
// ==========================================
// BULLMQ WORKER: SCRAPING & REVIEWS
// ==========================================
@@ -206,17 +226,17 @@ const worker = new Worker('scrape-jobs', async (job) => {
});
let targetUrl = cleanBookingUrl(url);
if (isReviewScan && !targetUrl.includes('#tab-reviews')) {
targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews';
}
console.log(`[Job ${job.id}] 🌐 Bereinigte Ziel-URL: ${targetUrl}`);
await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 45000 });
await page.waitForLoadState('domcontentloaded').catch(() => {});
await page.waitForTimeout(2000);
// Prüfen, ob AWS WAF Challenge aktiv ist
const isWafChallenge = await page.evaluate(() => {
// Prüfen, ob AWS WAF Challenge aktiv ist (sicher gegenüber gleichzeitiger Navigation)
let isWafChallenge = false;
try {
isWafChallenge = await page.evaluate(() => {
return Boolean(
document.querySelector('#challenge-container') ||
document.querySelector('script[src*="challenge.js"]') ||
@@ -224,6 +244,21 @@ const worker = new Worker('scrape-jobs', async (job) => {
(document.title === '' && (document.body ? document.body.innerText.includes('verify that you') : false))
);
});
} catch (navErr) {
await page.waitForLoadState('domcontentloaded').catch(() => {});
await page.waitForTimeout(2500);
try {
isWafChallenge = await page.evaluate(() => {
return Boolean(
document.querySelector('#challenge-container') ||
document.querySelector('script[src*="challenge.js"]') ||
window.AwsWafIntegration
);
});
} catch (e) {
isWafChallenge = false;
}
}
if (isWafChallenge) {
console.log(`[Job ${job.id}] 🛡️ AWS WAF Challenge erkannt. Warte auf automatische Token-Weiterleitung...`);
@@ -234,12 +269,14 @@ const worker = new Worker('scrape-jobs', async (job) => {
return !hasChallenge || Boolean(hasContent);
}, { timeout: 25000 });
console.log(`[Job ${job.id}] 🔓 Weiterleitung nach AWS WAF erfolgt!`);
await page.waitForLoadState('domcontentloaded').catch(() => {});
await page.waitForTimeout(2000);
} catch (wafErr) {
console.log(`[Job ${job.id}] ⚠️ AWS WAF Timeout. Versuche direkten Reload der Basis-URL...`);
try {
const parsed = new URL(targetUrl);
await page.goto(parsed.origin + parsed.pathname, { waitUntil: 'domcontentloaded', timeout: 30000 });
await page.waitForLoadState('domcontentloaded').catch(() => {});
await page.waitForTimeout(3000);
} catch (reloadErr) {}
}
@@ -290,7 +327,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
} catch (e) {}
// 2. STAMMDATEN EXTRAHIEREN (JSON-LD + DOM + META FALLBACKS)
hotelDetails = await page.evaluate(() => {
hotelDetails = await safeEvaluate(page, () => {
let name = null;
let rating = null;
let reviewCount = null;
@@ -450,7 +487,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
}
return { name, rating, reviewCount, address, image };
});
}, { name: null, rating: null, reviewCount: null, address: null, image: null });
// Fallback: Name aus URL-Slug ableiten wenn im HTML nichts gefunden wurde
if (!hotelDetails.name) {
@@ -458,7 +495,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
}
// 3. DOM-DATEN (Nächte, Personen, Zimmergröße)
const extraDom = await page.evaluate(() => {
const extraDom = await safeEvaluate(page, () => {
try {
const bodyText = document.body ? (document.body.innerText || '') : '';
const nMatch = bodyText.match(/(\d+)\s*Nächte?/i);
@@ -472,14 +509,14 @@ const worker = new Worker('scrape-jobs', async (job) => {
} catch (e) {
return { dNights: 1, dPersons: 2, roomSize: null };
}
});
}, { dNights: 1, dPersons: 2, roomSize: null });
domData.dNights = extraDom.dNights;
domData.dPersons = extraDom.dPersons;
roomSizeM2 = extraDom.roomSize;
// PREIS AUSLESEN
scrapedPrice = await page.evaluate(() => {
scrapedPrice = await safeEvaluate(page, () => {
try {
const selectors = [
'[data-testid="price-and-discounted-price"]',
@@ -502,10 +539,10 @@ const worker = new Worker('scrape-jobs', async (job) => {
}
} catch (e) {}
return null;
});
}, null);
// GENIUS BADGE PRÜFUNG
hasGenius = await page.evaluate(() => {
hasGenius = await safeEvaluate(page, () => {
try {
const geniusSelectors = [
'.m-badge__genius',
@@ -519,7 +556,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
} catch (e) {
return false;
}
});
}, false);
console.log(`[Job ${job.id}] 📋 Extrahierte Stammdaten:`, {
name: hotelDetails.name,
@@ -560,59 +597,38 @@ const worker = new Worker('scrape-jobs', async (job) => {
}
} catch (e) {}
// B. 500px nach unten scrollen, damit Bookings Sticky-Header (#reviews-tab-trigger) überhaupt sichtbar & aktiv wird
await page.evaluate(() => window.scrollBy(0, 550));
await page.waitForTimeout(1500);
// B. Herunterscrollen um Lazy-Components zu hydrieren und Review-Dialog zu öffnen
console.log(`[Job ${job.id}] 📜 Scrolle herunter, um verzögert geladene Review-Komponenten zu aktivieren...`);
await page.evaluate(() => window.scrollTo(0, 4000));
await page.waitForTimeout(2000);
// C. Klick per echtem Maus-Event auf den Trigger ausführen
let clicked = false;
// C. Klick auf Review-Trigger ausführen
let isPanelOpen = false;
try {
// Liste möglicher Klick-Ziele (Header-Tab, Scorecard-Badge oder Text "Gästebewertungen")
const triggerLocators = [
page.locator('#reviews-tab-trigger').first(),
page.locator('[data-testid="Property-Header-Nav-Tab-Trigger-reviews"]').first(),
page.locator('a:has-text("Gästebewertungen")').first(),
page.locator('button:has-text("Gästebewertungen")').first(),
page.locator('[data-testid="review-score-right-component"]').first(),
page.locator('[data-testid="review-score-component"]').first()
];
for (const loc of triggerLocators) {
if (await loc.count() > 0 && await loc.isVisible()) {
console.log(`[Job ${job.id}] 👉 Sichtbaren Review-Trigger gefunden. Führe Klick aus...`);
// Reale Maus-Koordinaten ermitteln & klicken
const box = await loc.boundingBox();
if (box) {
await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2);
clicked = true;
await page.waitForTimeout(3500);
break;
} else {
await loc.click({ force: true }).catch(() => {});
clicked = true;
await page.waitForTimeout(3500);
break;
}
}
}
// Fallback: JS-Klick direkt auf das HTML-Element mit Text "Gästebewertungen"
if (!clicked) {
console.log(`[Job ${job.id}] ⚠️ Fallback: Klick per DOM-Suche nach 'Gästebewertungen'...`);
clicked = await page.evaluate(() => {
const el = Array.from(document.querySelectorAll('a, button, span, div'))
.find(e => e.innerText && e.innerText.includes('Gästebewertungen') && (e.id === 'reviews-tab-trigger' || e.closest('#reviews-tab-trigger') || e.getAttribute('data-testid')?.includes('reviews')));
if (el) {
const clickTarget = el.closest('a') || el.closest('button') || el;
clickTarget.click();
clickTarget.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
console.log(`[Job ${job.id}] 👉 Öffne Review-Dialog über 'Alle Bewertungen lesen' / 'Gästebewertungen'...`);
isPanelOpen = await page.evaluate(() => {
const btn = document.querySelector('button[data-testid="fr-read-all-reviews"]') ||
document.querySelector('button[data-testid="review-score-read-all-actionable"]') ||
document.getElementById('reviews-tab-trigger') ||
document.querySelector('[data-testid="Property-Header-Nav-Tab-Trigger-reviews"]');
if (btn) {
btn.scrollIntoView({ behavior: 'instant', block: 'center' });
btn.click();
btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
return true;
}
return false;
}).catch(() => false);
if (clicked) await page.waitForTimeout(3500);
// Warten bis das Dialog-Panel mit den 1.000+ Bewertungen gerendert ist
try {
await page.waitForSelector(
'div[role="dialog"] [data-testid="review-card"], #reviewCardsSection [data-testid="review-card"], [data-testid="review-card"]',
{ timeout: 15000 }
);
console.log(`[Job ${job.id}] 🌟 Review-Dialog ('role="dialog"') mit allen Bewertungen erfolgreich geladen!`);
} catch (waitErr) {
console.log(`[Job ${job.id}] ⚠️ Dialog-Karten nicht sofort erkannt, fahre mit vorhandenem DOM fort...`);
}
// Screenshot zur visuellen Kontrolle aktualisieren
@@ -625,40 +641,13 @@ const worker = new Worker('scrape-jobs', async (job) => {
console.log(`[Job ${job.id}] ⚠️ Fehler beim Klick auf Review-Trigger:`, e.message);
}
// Warten bis Review-Karten im Sliding Panel oder DOM sichtbar sind
try {
await page.waitForSelector(
'[data-testid="review-card"], [data-testid="review-cards"], #hp-reviews-sliding, div[data-component="hotel/reviews/review-card"], li.review_item',
{ timeout: 8000 }
);
console.log(`[Job ${job.id}] 👁️ Review-Bereich erfolgreich gerendert.`);
} catch (e) {
console.log(`[Job ${job.id}] ⚠️ Wartezeit abgelaufen – starte Auslesen aus dem aktuellen DOM...`);
}
// C. Sortierung auf "Neueste zuerst" (NEWEST_FIRST) umstellen
// D. Sortierung auf "Neueste zuerst" (NEWEST_FIRST) umstellen
try {
const sorterLocator = page.locator('#reviewListSorters, [data-testid="reviews-sorter-component"], select[name="reviewListSorters"]').first();
if (await sorterLocator.count() > 0) {
console.log(`[Job ${job.id}] 🔄 Stelle Sortierung auf 'NEWEST_FIRST' (Neueste zuerst) um...`);
await sorterLocator.selectOption('NEWEST_FIRST');
await page.waitForTimeout(2500); // Warten bis Booking die Liste per AJAX neu lädt
} else {
// Fallback via JavaScript Event-Dispatch
const changed = await page.evaluate(() => {
const select = document.querySelector('#reviewListSorters, [data-testid="reviews-sorter-component"], select[name="reviewListSorters"]');
if (select) {
select.value = 'NEWEST_FIRST';
select.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
return false;
}).catch(() => false);
if (changed) {
console.log(`[Job ${job.id}] 🔄 Sortierung via JS-Event auf 'NEWEST_FIRST' umgestellt.`);
await page.waitForTimeout(2500);
}
await page.waitForTimeout(3000); // Warten bis Booking die Liste per AJAX neu lädt
}
} catch (e) {
console.log(`[Job ${job.id}] ⚠️ Sortierung konnte nicht angepasst werden:`, e.message);
@@ -671,8 +660,10 @@ const worker = new Worker('scrape-jobs', async (job) => {
pageCount++;
console.log(`[Job ${job.id}] 📄 Scanne Bewertungsseite ${pageCount} (Bisher gesammelt: ${extractedReviews.length})...`);
const evalResult = await page.evaluate(({ westernCountries }) => {
const evalResult = await safeEvaluate(page, ({ westernCountries }) => {
let cards = Array.from(document.querySelectorAll([
'#reviewCardsSection [data-testid="review-card"]',
'div[role="dialog"] [data-testid="review-card"]',
'[data-testid="review-card"]',
'[data-testid="review-cards"] > li',
'[data-testid="review-cards"] > div',
@@ -681,20 +672,24 @@ const worker = new Worker('scrape-jobs', async (job) => {
'li.review_list_new_item_block',
'div.c-review-block__item',
'div[itemprop="review"]',
'li.review_item'
'li.review_item',
'div.c-review',
'div.review_item',
'.c-review-block',
'li.c-review-block'
].join(',')));
if (cards.length === 0) {
const container = document.querySelector('#hp-reviews-sliding, #reviewCardsSection, [data-testid="review-cards"], #blockdisplay4');
const container = document.querySelector('#hp-reviews-sliding, #reviewCardsSection, [data-testid="review-cards"], div[role="dialog"]');
if (container) {
cards = Array.from(container.querySelectorAll('li, div')).filter(el => (el.innerText || '').length > 40);
cards = Array.from(container.querySelectorAll('li, div.c-review-block, div.c-review')).filter(el => (el.innerText || '').length > 30);
}
}
// Filterboxen und Scorecard-Zusammenfassungen explizit ausschließen
cards = cards.filter(el => {
const text = el.innerText || '';
return text.length > 25 &&
return text.length > 20 &&
!text.includes('Wählen Sie Themen') &&
!text.includes('Kategorien:') &&
!el.classList.contains('review_score_breakdown');
@@ -710,15 +705,18 @@ const worker = new Worker('scrape-jobs', async (job) => {
if (!cardText.trim()) continue;
// 1. Herkunftsland des Gastes ermitteln
const countryEl = card.querySelector('[data-testid="review-country"], .bui-avatar-or-image__title, .user_country, .c-review__country');
const guestCountryText = countryEl ? countryEl.innerText : cardText;
const flagImg = card.querySelector('[data-testid="review-avatar"] .fff1944c52 img[alt], [data-testid="review-country"] img[alt], img.b8d1620349');
const countrySpan = card.querySelector('[data-testid="review-avatar"] .fff1944c52 span:not(.cec4c9cfef), [data-testid="review-country"]');
const guestCountryText = (flagImg ? flagImg.getAttribute('alt') : (countrySpan ? countrySpan.innerText : '')).trim();
if (guestCountryText && guestCountryText.length > 1) {
const isWestern = westernCountries.some(c => guestCountryText.toLowerCase().includes(c.toLowerCase()));
if (!isWestern) {
const preview = guestCountryText.substring(0, 50).replace(/\n/g, ' ');
debugLogs.push(`Karte #${idx + 1}: ❌ Land nicht in Liste ("${preview}")`);
continue;
}
}
// 2. Datum / Jahr prüfen (letzte 2 Jahre)
const yearMatch = cardText.match(/\b(20\d{2})\b/);
@@ -737,8 +735,8 @@ const worker = new Worker('scrape-jobs', async (job) => {
const dateEl = card.querySelector('[data-testid="review-date"], .c-review__date, .review_item_date');
const title = titleEl ? titleEl.innerText.trim() : '';
const positive = posEl ? posEl.innerText.trim() : '';
const negative = negEl ? negEl.innerText.trim() : '';
const positive = posEl ? posEl.innerText.trim().replace(/^„|“$/g, '').trim() : '';
const negative = negEl ? negEl.innerText.trim().replace(/^„|“$/g, '').trim() : '';
const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell');
let fallbackText = '';
@@ -757,7 +755,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
results,
debugLogs: debugLogs.slice(0, 8)
};
}, { westernCountries: WESTERN_COUNTRIES });
}, { totalCards: 0, results: [], debugLogs: [] }, { westernCountries: WESTERN_COUNTRIES });
console.log(`[Job ${job.id}] 🔍 Echte Gästekarten gefunden: ${evalResult.totalCards}`);
if (evalResult.debugLogs) {
@@ -778,7 +776,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
// Weiterblättern im Modal oder der Hauptseite
try {
const nextBtn = await page.$(
'#hp-reviews-sliding button[aria-label*="Nächste"], #hp-reviews-sliding button[aria-label*="Next"], #reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"], a.paging-next, li.bui-pagination__next-arrow a'
'div[role="dialog"] button[aria-label="Nächste Seite"], button[aria-label="Nächste Seite"], button[aria-label*="Nächste Seite"], #hp-reviews-sliding button[aria-label*="Nächste"], button[aria-label*="Next"], #reviewCardsSection button:has(svg), a.paging-next, li.bui-pagination__next-arrow a'
);
if (nextBtn) {
const isDisabled = await nextBtn.getAttribute('disabled');

BIN
src/review-debug.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 652 KiB