Review fix
This commit is contained in:
355
src/index.js
355
src/index.js
@@ -2,6 +2,9 @@ import { Worker } from 'bullmq';
|
|||||||
import { chromium } from 'playwright';
|
import { chromium } from 'playwright';
|
||||||
import axios from 'axios';
|
import axios from 'axios';
|
||||||
|
|
||||||
|
// ==========================================
|
||||||
|
// KONFIGURATION & LISTEN
|
||||||
|
// ==========================================
|
||||||
const REDIS_CONFIG = {
|
const REDIS_CONFIG = {
|
||||||
connection: {
|
connection: {
|
||||||
host: process.env.REDIS_HOST || 'redis',
|
host: process.env.REDIS_HOST || 'redis',
|
||||||
@@ -12,37 +15,34 @@ const REDIS_CONFIG = {
|
|||||||
const SERPAPI_KEY = process.env.SERPAPI_KEY || null;
|
const SERPAPI_KEY = process.env.SERPAPI_KEY || null;
|
||||||
|
|
||||||
const WESTERN_COUNTRIES = [
|
const WESTERN_COUNTRIES = [
|
||||||
'Deutschland', 'Schweiz', 'Österreich', 'Vereinigte Staaten', 'USA',
|
'Deutschland', 'Schweiz', 'Österreich', 'Germany', 'Switzerland', 'Austria',
|
||||||
'Vereinigtes Königreich', 'Großbritannien', 'Niederlande', 'Frankreich',
|
'Vereinigte Staaten', 'USA', 'United States', 'Vereinigtes Königreich', 'Großbritannien',
|
||||||
'Spanien', 'Italien', 'Kanada', 'Australien', 'Belgien', 'Schweden',
|
'United Kingdom', 'UK', 'Niederlande', 'Netherlands', 'Frankreich', 'France',
|
||||||
'Norwegen', 'Dänemark', 'Finnland', 'Irland', 'Luxemburg', 'Indien'
|
'Spanien', 'Spain', 'Italien', 'Italy', 'Kanada', 'Canada', 'Australien', 'Australia',
|
||||||
|
'Belgien', 'Belgium', 'Schweden', 'Sweden', 'Norwegen', 'Norway', 'Dänemark', 'Denmark',
|
||||||
|
'Finnland', 'Finland', 'Irland', 'Ireland', 'Luxemburg', 'Luxembourg', 'Indien', 'India'
|
||||||
];
|
];
|
||||||
|
|
||||||
// 1. Google Maps Fotos via SerpApi abrufen (inkl. Adressbereinigung & Fallback)
|
// ==========================================
|
||||||
|
// HILFSFUNKTION: GOOGLE MAPS BAD-FOTOS VIA SERPAPI
|
||||||
|
// ==========================================
|
||||||
async function getGoogleBathroomPhotosViaApi(hotelName, hotelAddress) {
|
async function getGoogleBathroomPhotosViaApi(hotelName, hotelAddress) {
|
||||||
if (!SERPAPI_KEY) {
|
if (!SERPAPI_KEY) {
|
||||||
console.log('⚠️ [SerpApi] Kein SERPAPI_KEY in der .env hinterlegt!');
|
console.log('ℹ️ [SerpApi] Kein SERPAPI_KEY konfiguriert – überspringe Google-Maps-Fotos.');
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Booking-Adresse bereinigen: Nur Stadt und Land nutzen für maximale Trefferquote
|
|
||||||
let cleanLocation = '';
|
let cleanLocation = '';
|
||||||
if (hotelAddress) {
|
if (hotelAddress) {
|
||||||
const parts = hotelAddress.split(',').map(p => p.trim());
|
const parts = hotelAddress.split(',').map(p => p.trim());
|
||||||
cleanLocation = parts.length > 1 ? parts.slice(-2).join(' ') : hotelAddress;
|
cleanLocation = parts.length > 1 ? parts.slice(-2).join(' ') : hotelAddress;
|
||||||
}
|
}
|
||||||
const query = `${hotelName} ${cleanLocation}`.trim();
|
const query = `${hotelName} ${cleanLocation}`.trim();
|
||||||
console.log(`🔍 [SerpApi] Suche Google Maps Place für: "${query}"...`);
|
console.log(`🔍 [SerpApi] Suche Google Maps Ort für: "${query}"...`);
|
||||||
|
|
||||||
// A: Google Maps Ort suchen
|
|
||||||
const searchRes = await axios.get('https://serpapi.com/search.json', {
|
const searchRes = await axios.get('https://serpapi.com/search.json', {
|
||||||
params: {
|
params: { engine: 'google_maps', q: query, hl: 'de', api_key: SERPAPI_KEY },
|
||||||
engine: 'google_maps',
|
|
||||||
q: query,
|
|
||||||
hl: 'de',
|
|
||||||
api_key: SERPAPI_KEY
|
|
||||||
},
|
|
||||||
timeout: 15000
|
timeout: 15000
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -55,73 +55,53 @@ async function getGoogleBathroomPhotosViaApi(hotelName, hotelAddress) {
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`📸 [SerpApi] data_id gefunden (${dataId}). Lade Fotoliste...`);
|
|
||||||
|
|
||||||
// B: Fotos & Kategorien abrufen
|
|
||||||
const photosRes = await axios.get('https://serpapi.com/search.json', {
|
const photosRes = await axios.get('https://serpapi.com/search.json', {
|
||||||
params: {
|
params: { engine: 'google_maps_photos', data_id: dataId, hl: 'de', api_key: SERPAPI_KEY },
|
||||||
engine: 'google_maps_photos',
|
|
||||||
data_id: dataId,
|
|
||||||
hl: 'de',
|
|
||||||
api_key: SERPAPI_KEY
|
|
||||||
},
|
|
||||||
timeout: 15000
|
timeout: 15000
|
||||||
});
|
});
|
||||||
|
|
||||||
const categories = photosRes.data.categories || [];
|
const categories = photosRes.data.categories || [];
|
||||||
// Flexible Erkennung für Bad-, Zimmer- und Innen-Tags
|
const strictBathKeywords = ['badezimmer', 'bad', 'bathroom', 'wc', 'toilet', 'washroom', 'restroom'];
|
||||||
const bathCat = categories.find(c => {
|
const bathCat = categories.find(c => {
|
||||||
const title = (c.title || '').toLowerCase();
|
const title = (c.title || '').toLowerCase().trim();
|
||||||
return title.includes('bad') || title.includes('zimmer') || title.includes('room') || title.includes('innen') || title.includes('gast');
|
return strictBathKeywords.some(kw => title.includes(kw));
|
||||||
});
|
});
|
||||||
|
|
||||||
let photosList = photosRes.data.photos || [];
|
let photosList = [];
|
||||||
|
|
||||||
// Falls Kategorie vorhanden, gezielt Unterkategorie laden
|
|
||||||
if (bathCat?.id) {
|
if (bathCat?.id) {
|
||||||
console.log(`🚽 [SerpApi] Kategorie gefunden: "${bathCat.title}"`);
|
console.log(`🚽 [SerpApi] Bad-Kategorie gefunden: "${bathCat.title}"`);
|
||||||
try {
|
|
||||||
const catRes = await axios.get('https://serpapi.com/search.json', {
|
const catRes = await axios.get('https://serpapi.com/search.json', {
|
||||||
params: {
|
params: { engine: 'google_maps_photos', data_id: dataId, category_id: bathCat.id, hl: 'de', api_key: SERPAPI_KEY },
|
||||||
engine: 'google_maps_photos',
|
|
||||||
data_id: dataId,
|
|
||||||
category_id: bathCat.id,
|
|
||||||
hl: 'de',
|
|
||||||
api_key: SERPAPI_KEY
|
|
||||||
},
|
|
||||||
timeout: 15000
|
timeout: 15000
|
||||||
});
|
});
|
||||||
if (catRes.data.photos && catRes.data.photos.length > 0) {
|
if (catRes.data.photos?.length > 0) photosList = catRes.data.photos;
|
||||||
photosList = catRes.data.photos;
|
} else {
|
||||||
}
|
console.log('ℹ️ [SerpApi] Kein separater Bad-Ordner auf Google Maps vorhanden.');
|
||||||
} catch (e) {
|
return [];
|
||||||
console.log('⚠️ [SerpApi] Fallback auf allgemeine Fotos aktiv.');
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// URLs bereinigen und auf HD-Auflösung setzen
|
return photosList
|
||||||
const imageUrls = photosList
|
|
||||||
.map(p => p.image || p.thumbnail)
|
.map(p => p.image || p.thumbnail)
|
||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
.map(url => url.replace(/=w\d+-h\d+[^=]*/, '=w1200-h800-k-no'))
|
.map(url => url.replace(/=w\d+-h\d+[^=]*/, '=w1200-h800-k-no'))
|
||||||
.slice(0, 12);
|
.slice(0, 12);
|
||||||
|
|
||||||
console.log(`✅ [SerpApi] ${imageUrls.length} Gästefotos erfolgreich geladen.`);
|
|
||||||
return imageUrls;
|
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('❌ [SerpApi] Fehler:', error.response?.data?.error || error.message);
|
console.error('❌ [SerpApi] Fehler beim Fotoabruf:', error.message);
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Worker Definition
|
// ==========================================
|
||||||
|
// BULLMQ WORKER: SCRAPING & REVIEWS
|
||||||
|
// ==========================================
|
||||||
const worker = new Worker('scrape-jobs', async (job) => {
|
const worker = new Worker('scrape-jobs', async (job) => {
|
||||||
const { url, webhookUrl, appWebhookUrl, hotelId } = job.data;
|
const { url, webhookUrl, appWebhookUrl, hotelId } = job.data;
|
||||||
const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http'));
|
const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http'));
|
||||||
console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews + Bad-Fotos)' : 'PREIS-SCAN (Schnell)'} für: ${url}`);
|
|
||||||
|
|
||||||
// Alle Variablen fest vorinitialisieren
|
console.log(`\n==================================================`);
|
||||||
|
console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews + Fotos)' : 'PREIS-SCAN (Schnell)'} für: ${url}`);
|
||||||
|
console.log(`==================================================`);
|
||||||
|
|
||||||
let scrapedPrice = null;
|
let scrapedPrice = null;
|
||||||
let hasGenius = false;
|
let hasGenius = false;
|
||||||
let roomSizeM2 = null;
|
let roomSizeM2 = null;
|
||||||
@@ -132,21 +112,37 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
|||||||
let browser = null;
|
let browser = null;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
browser = await chromium.launch({ headless: true });
|
// 1. Browser mit Anti-Bot & Stealth-Optionen starten
|
||||||
|
browser = await chromium.launch({
|
||||||
|
headless: true,
|
||||||
|
args: [
|
||||||
|
'--no-sandbox',
|
||||||
|
'--disable-setuid-sandbox',
|
||||||
|
'--disable-blink-features=AutomationControlled',
|
||||||
|
'--disable-infobars'
|
||||||
|
]
|
||||||
|
});
|
||||||
|
|
||||||
const context = await browser.newContext({
|
const context = await browser.newContext({
|
||||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
|
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
|
||||||
viewport: { width: 1280, height: 900 },
|
viewport: { width: 1366, height: 900 },
|
||||||
locale: 'de-DE',
|
locale: 'de-DE',
|
||||||
timezoneId: 'Europe/Berlin'
|
timezoneId: 'Europe/Berlin'
|
||||||
});
|
});
|
||||||
|
|
||||||
const page = await context.newPage();
|
const page = await context.newPage();
|
||||||
|
|
||||||
|
// Verhindert Erkennung über navigator.webdriver
|
||||||
|
await page.addInitScript(() => {
|
||||||
|
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||||
|
});
|
||||||
|
|
||||||
let targetUrl = url;
|
let targetUrl = url;
|
||||||
if (isReviewScan && !targetUrl.includes('#tab-reviews')) {
|
if (isReviewScan && !targetUrl.includes('#tab-reviews')) {
|
||||||
targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews';
|
targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews';
|
||||||
}
|
}
|
||||||
|
|
||||||
await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 35000 });
|
await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 40000 });
|
||||||
await page.waitForTimeout(2500);
|
await page.waitForTimeout(2500);
|
||||||
|
|
||||||
// DATUM AUS URL PARSEN
|
// DATUM AUS URL PARSEN
|
||||||
@@ -156,7 +152,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
|||||||
domData.checkoutDate = parsedUrl.searchParams.get('checkout');
|
domData.checkoutDate = parsedUrl.searchParams.get('checkout');
|
||||||
} catch (e) {}
|
} catch (e) {}
|
||||||
|
|
||||||
// 1. JSON-LD STAMMDATEN
|
// 2. JSON-LD STAMMDATEN EXTRAHIEREN
|
||||||
hotelDetails = await page.evaluate(() => {
|
hotelDetails = await page.evaluate(() => {
|
||||||
try {
|
try {
|
||||||
const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
|
const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
|
||||||
@@ -178,7 +174,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
|||||||
return { name: null, rating: null, reviewCount: null, address: null, image: null };
|
return { name: null, rating: null, reviewCount: null, address: null, image: null };
|
||||||
});
|
});
|
||||||
|
|
||||||
// 2. DOM DATEN
|
// 3. DOM-DATEN (Nächte, Personen, Zimmergröße)
|
||||||
const extraDom = await page.evaluate(() => {
|
const extraDom = await page.evaluate(() => {
|
||||||
try {
|
try {
|
||||||
const bodyText = document.body ? (document.body.innerText || '') : '';
|
const bodyText = document.body ? (document.body.innerText || '') : '';
|
||||||
@@ -214,7 +210,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
|||||||
return null;
|
return null;
|
||||||
});
|
});
|
||||||
|
|
||||||
// GENIUS BADGE ERKENNUNG
|
// GENIUS BADGE PRÜFUNG
|
||||||
hasGenius = await page.evaluate(() => {
|
hasGenius = await page.evaluate(() => {
|
||||||
const geniusSelectors = [
|
const geniusSelectors = [
|
||||||
'.m-badge__genius',
|
'.m-badge__genius',
|
||||||
@@ -231,44 +227,175 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
|||||||
console.log(`[Job ${job.id}] 💙 Genius-Badge erkannt (-10% Rabatt gilt)!`);
|
console.log(`[Job ${job.id}] 💙 Genius-Badge erkannt (-10% Rabatt gilt)!`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. KOMMENTARE EXTRAHIEREN
|
// 4. REVIEWS EXTRAHIEREN (STICKY-HEADER AKTIVIERUNG & ECHTER MAUSKLICK)
|
||||||
if (isReviewScan) {
|
if (isReviewScan) {
|
||||||
console.log(`[Job ${job.id}] Starte Review-Extraction mit Pagination...`);
|
console.log(`[Job ${job.id}] 🚀 Starte gezielte Review-Extraction für "${hotelDetails.name || 'Hotel'}"...`);
|
||||||
|
|
||||||
|
// A. Cookie-Banner und störende Overlays schließen
|
||||||
try {
|
try {
|
||||||
const reviewTab = await page.$('a[href*="#tab-reviews"], #js--hp-gallery-scorecard a, [data-testid="review-score-link"]');
|
const dismissSelectors = [
|
||||||
if (reviewTab) {
|
'#onetrust-accept-btn-handler',
|
||||||
await reviewTab.click();
|
'button#onetrust-reject-all-handler',
|
||||||
await page.waitForTimeout(2000);
|
'button[data-testid="selection-modal-close-btn"]',
|
||||||
|
'button[aria-label="Schließen"]',
|
||||||
|
'button[aria-label="Close"]'
|
||||||
|
];
|
||||||
|
for (const btnSel of dismissSelectors) {
|
||||||
|
const btn = await page.$(btnSel);
|
||||||
|
if (btn) {
|
||||||
|
await btn.click().catch(() => {});
|
||||||
|
await page.waitForTimeout(400);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} catch (e) {}
|
} catch (e) {}
|
||||||
|
|
||||||
await page.evaluate(() => window.scrollBy(0, 1000));
|
// B. 500px nach unten scrollen, damit Bookings Sticky-Header (#reviews-tab-trigger) überhaupt sichtbar & aktiv wird
|
||||||
await page.waitForTimeout(2000);
|
await page.evaluate(() => window.scrollBy(0, 550));
|
||||||
|
await page.waitForTimeout(1500);
|
||||||
|
|
||||||
|
// C. Klick per echtem Maus-Event auf den Trigger ausführen
|
||||||
|
let clicked = false;
|
||||||
|
try {
|
||||||
|
// Liste möglicher Klick-Ziele (Header-Tab, Scorecard-Badge oder Text "Gästebewertungen")
|
||||||
|
const triggerLocators = [
|
||||||
|
page.locator('#reviews-tab-trigger').first(),
|
||||||
|
page.locator('[data-testid="Property-Header-Nav-Tab-Trigger-reviews"]').first(),
|
||||||
|
page.locator('a:has-text("Gästebewertungen")').first(),
|
||||||
|
page.locator('button:has-text("Gästebewertungen")').first(),
|
||||||
|
page.locator('[data-testid="review-score-right-component"]').first(),
|
||||||
|
page.locator('[data-testid="review-score-component"]').first()
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const loc of triggerLocators) {
|
||||||
|
if (await loc.count() > 0 && await loc.isVisible()) {
|
||||||
|
console.log(`[Job ${job.id}] 👉 Sichtbaren Review-Trigger gefunden. Führe Klick aus...`);
|
||||||
|
|
||||||
|
// Reale Maus-Koordinaten ermitteln & klicken
|
||||||
|
const box = await loc.boundingBox();
|
||||||
|
if (box) {
|
||||||
|
await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2);
|
||||||
|
clicked = true;
|
||||||
|
await page.waitForTimeout(3500);
|
||||||
|
break;
|
||||||
|
} else {
|
||||||
|
await loc.click({ force: true }).catch(() => {});
|
||||||
|
clicked = true;
|
||||||
|
await page.waitForTimeout(3500);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: JS-Klick direkt auf das HTML-Element mit Text "Gästebewertungen"
|
||||||
|
if (!clicked) {
|
||||||
|
console.log(`[Job ${job.id}] ⚠️ Fallback: Klick per DOM-Suche nach 'Gästebewertungen'...`);
|
||||||
|
clicked = await page.evaluate(() => {
|
||||||
|
const el = Array.from(document.querySelectorAll('a, button, span, div'))
|
||||||
|
.find(e => e.innerText && e.innerText.includes('Gästebewertungen') && (e.id === 'reviews-tab-trigger' || e.closest('#reviews-tab-trigger') || e.getAttribute('data-testid')?.includes('reviews')));
|
||||||
|
if (el) {
|
||||||
|
const clickTarget = el.closest('a') || el.closest('button') || el;
|
||||||
|
clickTarget.click();
|
||||||
|
clickTarget.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}).catch(() => false);
|
||||||
|
|
||||||
|
if (clicked) await page.waitForTimeout(3500);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Screenshot zur visuellen Kontrolle aktualisieren
|
||||||
|
try {
|
||||||
|
await page.screenshot({ path: 'src/review-debug.png', fullPage: false });
|
||||||
|
console.log(`[Job ${job.id}] 📸 Screenshot aktualisiert unter: scraper-service/src/review-debug.png`);
|
||||||
|
} catch (e) {}
|
||||||
|
|
||||||
|
} catch (e) {
|
||||||
|
console.log(`[Job ${job.id}] ⚠️ Fehler beim Klick auf Review-Trigger:`, e.message);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Warten bis Review-Karten im Sliding Panel oder DOM sichtbar sind
|
||||||
|
try {
|
||||||
|
await page.waitForSelector(
|
||||||
|
'[data-testid="review-card"], [data-testid="review-cards"], #hp-reviews-sliding, div[data-component="hotel/reviews/review-card"], li.review_item',
|
||||||
|
{ timeout: 8000 }
|
||||||
|
);
|
||||||
|
console.log(`[Job ${job.id}] 👁️ Review-Bereich erfolgreich gerendert.`);
|
||||||
|
} catch (e) {
|
||||||
|
console.log(`[Job ${job.id}] ⚠️ Wartezeit abgelaufen – starte Auslesen aus dem aktuellen DOM...`);
|
||||||
|
}
|
||||||
|
|
||||||
let pageCount = 0;
|
let pageCount = 0;
|
||||||
const seenKeys = new Set();
|
const seenKeys = new Set();
|
||||||
|
|
||||||
while (extractedReviews.length < 80 && pageCount < 15) {
|
while (pageCount < 30) {
|
||||||
const pageReviews = await page.evaluate(({ westernCountries }) => {
|
pageCount++;
|
||||||
const cards = Array.from(document.querySelectorAll('[data-testid="review-card"], [role="group"][aria-label*="Bewertung"], li.review_list_new_item_block, .review_item'));
|
console.log(`[Job ${job.id}] 📄 Scanne Bewertungsseite ${pageCount} (Bisher gesammelt: ${extractedReviews.length})...`);
|
||||||
const results = [];
|
|
||||||
const currentYear = 2026;
|
|
||||||
for (const card of cards) {
|
|
||||||
const cardText = card ? (card.innerText || '') : '';
|
|
||||||
if (!cardText) continue;
|
|
||||||
const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase()));
|
|
||||||
if (!isWestern) continue;
|
|
||||||
|
|
||||||
|
const evalResult = await page.evaluate(({ westernCountries }) => {
|
||||||
|
let cards = Array.from(document.querySelectorAll([
|
||||||
|
'[data-testid="review-card"]',
|
||||||
|
'[data-testid="review-cards"] > li',
|
||||||
|
'[data-testid="review-cards"] > div',
|
||||||
|
'div[data-testid*="review-card"]',
|
||||||
|
'li[data-testid*="review-card"]',
|
||||||
|
'li.review_list_new_item_block',
|
||||||
|
'div.c-review-block__item',
|
||||||
|
'div[itemprop="review"]',
|
||||||
|
'li.review_item'
|
||||||
|
].join(',')));
|
||||||
|
|
||||||
|
if (cards.length === 0) {
|
||||||
|
const container = document.querySelector('#hp-reviews-sliding, #reviewCardsSection, [data-testid="review-cards"], #blockdisplay4');
|
||||||
|
if (container) {
|
||||||
|
cards = Array.from(container.querySelectorAll('li, div')).filter(el => (el.innerText || '').length > 40);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filterboxen und Scorecard-Zusammenfassungen explizit ausschließen
|
||||||
|
cards = cards.filter(el => {
|
||||||
|
const text = el.innerText || '';
|
||||||
|
return text.length > 25 &&
|
||||||
|
!text.includes('Wählen Sie Themen') &&
|
||||||
|
!text.includes('Kategorien:') &&
|
||||||
|
!el.classList.contains('review_score_breakdown');
|
||||||
|
});
|
||||||
|
|
||||||
|
const results = [];
|
||||||
|
const debugLogs = [];
|
||||||
|
const currentYear = 2026;
|
||||||
|
|
||||||
|
for (let idx = 0; idx < cards.length; idx++) {
|
||||||
|
const card = cards[idx];
|
||||||
|
const cardText = card.innerText || '';
|
||||||
|
if (!cardText.trim()) continue;
|
||||||
|
|
||||||
|
// 1. Herkunftsland des Gastes ermitteln
|
||||||
|
const countryEl = card.querySelector('[data-testid="review-country"], .bui-avatar-or-image__title, .user_country, .c-review__country');
|
||||||
|
const guestCountryText = countryEl ? countryEl.innerText : cardText;
|
||||||
|
|
||||||
|
const isWestern = westernCountries.some(c => guestCountryText.toLowerCase().includes(c.toLowerCase()));
|
||||||
|
if (!isWestern) {
|
||||||
|
const preview = guestCountryText.substring(0, 50).replace(/\n/g, ' ');
|
||||||
|
debugLogs.push(`Karte #${idx + 1}: ❌ Land nicht in Liste ("${preview}")`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Datum / Jahr prüfen (letzte 2 Jahre)
|
||||||
const yearMatch = cardText.match(/\b(20\d{2})\b/);
|
const yearMatch = cardText.match(/\b(20\d{2})\b/);
|
||||||
if (yearMatch) {
|
if (yearMatch) {
|
||||||
const reviewYear = parseInt(yearMatch[1], 10);
|
const reviewYear = parseInt(yearMatch[1], 10);
|
||||||
if (currentYear - reviewYear > 2) continue;
|
if (currentYear - reviewYear > 2) {
|
||||||
|
debugLogs.push(`Karte #${idx + 1}: ❌ Zu alt (${reviewYear})`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const titleEl = card.querySelector('[data-testid="review-title"], h4');
|
// 3. Texte extrahieren
|
||||||
const posEl = card.querySelector('[data-testid="review-positive-text"], .review_pos');
|
const titleEl = card.querySelector('[data-testid="review-title"], .c-review__title, h4, .review_title');
|
||||||
const negEl = card.querySelector('[data-testid="review-negative-text"], .review_neg');
|
const posEl = card.querySelector('[data-testid="review-positive-text"], .c-review__row--positive, .review_pos');
|
||||||
const dateEl = card.querySelector('[data-testid="review-date"]');
|
const negEl = card.querySelector('[data-testid="review-negative-text"], .c-review__row--negative, .review_neg');
|
||||||
|
const dateEl = card.querySelector('[data-testid="review-date"], .c-review__date, .review_item_date');
|
||||||
|
|
||||||
const title = titleEl ? titleEl.innerText.trim() : '';
|
const title = titleEl ? titleEl.innerText.trim() : '';
|
||||||
const positive = posEl ? posEl.innerText.trim() : '';
|
const positive = posEl ? posEl.innerText.trim() : '';
|
||||||
@@ -276,16 +403,30 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
|||||||
const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell');
|
const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell');
|
||||||
|
|
||||||
let fallbackText = '';
|
let fallbackText = '';
|
||||||
if (!positive && !negative) fallbackText = cardText.replace(title, '').substring(0, 300).trim();
|
if (!positive && !negative) {
|
||||||
|
fallbackText = cardText.replace(title, '').substring(0, 300).trim();
|
||||||
|
}
|
||||||
|
|
||||||
if (title || positive || negative || fallbackText) {
|
if (title || positive || negative || fallbackText) {
|
||||||
results.push({ title, positive: positive || fallbackText, negative, date: dateText });
|
results.push({ title, positive: positive || fallbackText, negative, date: dateText });
|
||||||
|
debugLogs.push(`Karte #${idx + 1}: ✅ Akzeptiert ("${title || (positive ? positive.substring(0, 25) : 'Review')}")`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return results;
|
|
||||||
}, { westernCountries: WESTERN_COUNTRIES }).catch(() => []);
|
|
||||||
|
|
||||||
for (const rev of pageReviews) {
|
return {
|
||||||
|
totalCards: cards.length,
|
||||||
|
results,
|
||||||
|
debugLogs: debugLogs.slice(0, 8)
|
||||||
|
};
|
||||||
|
}, { westernCountries: WESTERN_COUNTRIES });
|
||||||
|
|
||||||
|
console.log(`[Job ${job.id}] 🔍 Echte Gästekarten gefunden: ${evalResult.totalCards}`);
|
||||||
|
if (evalResult.debugLogs) {
|
||||||
|
evalResult.debugLogs.forEach(log => console.log(`[Job ${job.id}] └─ ${log}`));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deduplizierung
|
||||||
|
for (const rev of evalResult.results) {
|
||||||
const key = `${rev.title}-${rev.positive}-${rev.date}`;
|
const key = `${rev.title}-${rev.positive}-${rev.date}`;
|
||||||
if (!seenKeys.has(key) && extractedReviews.length < 80) {
|
if (!seenKeys.has(key) && extractedReviews.length < 80) {
|
||||||
seenKeys.add(key);
|
seenKeys.add(key);
|
||||||
@@ -295,26 +436,32 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
|||||||
|
|
||||||
if (extractedReviews.length >= 80) break;
|
if (extractedReviews.length >= 80) break;
|
||||||
|
|
||||||
|
// Weiterblättern im Modal oder der Hauptseite
|
||||||
try {
|
try {
|
||||||
const nextBtn = await page.$('#reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"]');
|
const nextBtn = await page.$(
|
||||||
|
'#hp-reviews-sliding button[aria-label*="Nächste"], #hp-reviews-sliding button[aria-label*="Next"], #reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"], a.paging-next, li.bui-pagination__next-arrow a'
|
||||||
|
);
|
||||||
if (nextBtn) {
|
if (nextBtn) {
|
||||||
const isDisabled = await nextBtn.getAttribute('disabled');
|
const isDisabled = await nextBtn.getAttribute('disabled');
|
||||||
if (isDisabled !== null) break;
|
if (isDisabled === null) {
|
||||||
await nextBtn.click();
|
await nextBtn.click({ force: true });
|
||||||
await page.waitForTimeout(2000);
|
await page.waitForTimeout(2500);
|
||||||
pageCount++;
|
|
||||||
} else break;
|
} else break;
|
||||||
} catch (e) { break; }
|
} else break;
|
||||||
|
} catch (e) {
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
console.log(`[Job ${job.id}] ${extractedReviews.length} Kommentare extrahiert.`);
|
}
|
||||||
|
|
||||||
|
console.log(`[Job ${job.id}] ✅ Insgesamt ${extractedReviews.length} passende Kommentare extrahiert.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`[Job ${job.id}] Fehler beim Scraping:`, error.message);
|
console.error(`[Job ${job.id}] ❌ Fehler beim Scraping:`, error.message);
|
||||||
} finally {
|
} finally {
|
||||||
if (browser) await browser.close();
|
if (browser) await browser.close();
|
||||||
|
|
||||||
// 4. Google Maps Fotos nur bei gezieltem KI-Review-Scan abrufen (spart SerpApi-Credits)
|
// 5. GOOGLE MAPS FOTOS NUR BEI KI-REVIEW SCAN ABRUFEN
|
||||||
if (isReviewScan && hotelDetails.name) {
|
if (isReviewScan && hotelDetails.name) {
|
||||||
try {
|
try {
|
||||||
bathroomImages = await getGoogleBathroomPhotosViaApi(hotelDetails.name, hotelDetails.address);
|
bathroomImages = await getGoogleBathroomPhotosViaApi(hotelDetails.name, hotelDetails.address);
|
||||||
@@ -324,7 +471,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 5. Preishistorie und Bad-Bilder an Travel-App Webhook senden
|
// 6. PREISHISTORIE + STAMMDATEN AN NEXT.JS WEBHOOK SENDEN
|
||||||
if (appWebhookUrl) {
|
if (appWebhookUrl) {
|
||||||
try {
|
try {
|
||||||
await axios.post(appWebhookUrl, {
|
await axios.post(appWebhookUrl, {
|
||||||
@@ -342,13 +489,13 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
|||||||
persons: domData.dPersons || 2,
|
persons: domData.dPersons || 2,
|
||||||
bathroomImages: Array.isArray(bathroomImages) && bathroomImages.length > 0 ? bathroomImages : undefined
|
bathroomImages: Array.isArray(bathroomImages) && bathroomImages.length > 0 ? bathroomImages : undefined
|
||||||
});
|
});
|
||||||
console.log(`[Job ${job.id}] ✅ Preishistorie in Postgres gespeichert!`);
|
console.log(`[Job ${job.id}] ✅ Preishistorie & Stammdaten in Postgres gespeichert!`);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie:`, e.message);
|
console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie-Webhook:`, e.message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 6. Reviews an n8n senden
|
// 7. REVIEWS AN N8N WEBHOOK SENDEN
|
||||||
if (isReviewScan && extractedReviews.length > 0) {
|
if (isReviewScan && extractedReviews.length > 0) {
|
||||||
try {
|
try {
|
||||||
await axios.post(webhookUrl, {
|
await axios.post(webhookUrl, {
|
||||||
@@ -359,7 +506,9 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
|||||||
timestamp: new Date().toISOString()
|
timestamp: new Date().toISOString()
|
||||||
});
|
});
|
||||||
console.log(`[Job ${job.id}] 🚀 ${extractedReviews.length} Reviews an n8n gesendet!`);
|
console.log(`[Job ${job.id}] 🚀 ${extractedReviews.length} Reviews an n8n gesendet!`);
|
||||||
} catch (e) {}
|
} catch (e) {
|
||||||
|
console.error(`[Job ${job.id}] ❌ Fehler beim Senden an n8n:`, e.message);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}, { ...REDIS_CONFIG, concurrency: 2 });
|
}, { ...REDIS_CONFIG, concurrency: 2 });
|
||||||
|
|||||||
BIN
src/review-debug.png
Normal file
BIN
src/review-debug.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 342 KiB |
Reference in New Issue
Block a user