Review fix
This commit is contained in:
361
src/index.js
361
src/index.js
@@ -2,6 +2,9 @@ import { Worker } from 'bullmq';
|
||||
import { chromium } from 'playwright';
|
||||
import axios from 'axios';
|
||||
|
||||
// ==========================================
|
||||
// KONFIGURATION & LISTEN
|
||||
// ==========================================
|
||||
const REDIS_CONFIG = {
|
||||
connection: {
|
||||
host: process.env.REDIS_HOST || 'redis',
|
||||
@@ -12,37 +15,34 @@ const REDIS_CONFIG = {
|
||||
const SERPAPI_KEY = process.env.SERPAPI_KEY || null;
|
||||
|
||||
const WESTERN_COUNTRIES = [
|
||||
'Deutschland', 'Schweiz', 'Österreich', 'Vereinigte Staaten', 'USA',
|
||||
'Vereinigtes Königreich', 'Großbritannien', 'Niederlande', 'Frankreich',
|
||||
'Spanien', 'Italien', 'Kanada', 'Australien', 'Belgien', 'Schweden',
|
||||
'Norwegen', 'Dänemark', 'Finnland', 'Irland', 'Luxemburg', 'Indien'
|
||||
'Deutschland', 'Schweiz', 'Österreich', 'Germany', 'Switzerland', 'Austria',
|
||||
'Vereinigte Staaten', 'USA', 'United States', 'Vereinigtes Königreich', 'Großbritannien',
|
||||
'United Kingdom', 'UK', 'Niederlande', 'Netherlands', 'Frankreich', 'France',
|
||||
'Spanien', 'Spain', 'Italien', 'Italy', 'Kanada', 'Canada', 'Australien', 'Australia',
|
||||
'Belgien', 'Belgium', 'Schweden', 'Sweden', 'Norwegen', 'Norway', 'Dänemark', 'Denmark',
|
||||
'Finnland', 'Finland', 'Irland', 'Ireland', 'Luxemburg', 'Luxembourg', 'Indien', 'India'
|
||||
];
|
||||
|
||||
// 1. Google Maps Fotos via SerpApi abrufen (inkl. Adressbereinigung & Fallback)
|
||||
// ==========================================
|
||||
// HILFSFUNKTION: GOOGLE MAPS BAD-FOTOS VIA SERPAPI
|
||||
// ==========================================
|
||||
async function getGoogleBathroomPhotosViaApi(hotelName, hotelAddress) {
|
||||
if (!SERPAPI_KEY) {
|
||||
console.log('⚠️ [SerpApi] Kein SERPAPI_KEY in der .env hinterlegt!');
|
||||
console.log('ℹ️ [SerpApi] Kein SERPAPI_KEY konfiguriert – überspringe Google-Maps-Fotos.');
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
// Booking-Adresse bereinigen: Nur Stadt und Land nutzen für maximale Trefferquote
|
||||
let cleanLocation = '';
|
||||
if (hotelAddress) {
|
||||
const parts = hotelAddress.split(',').map(p => p.trim());
|
||||
cleanLocation = parts.length > 1 ? parts.slice(-2).join(' ') : hotelAddress;
|
||||
}
|
||||
const query = `${hotelName} ${cleanLocation}`.trim();
|
||||
console.log(`🔍 [SerpApi] Suche Google Maps Place für: "${query}"...`);
|
||||
console.log(`🔍 [SerpApi] Suche Google Maps Ort für: "${query}"...`);
|
||||
|
||||
// A: Google Maps Ort suchen
|
||||
const searchRes = await axios.get('https://serpapi.com/search.json', {
|
||||
params: {
|
||||
engine: 'google_maps',
|
||||
q: query,
|
||||
hl: 'de',
|
||||
api_key: SERPAPI_KEY
|
||||
},
|
||||
params: { engine: 'google_maps', q: query, hl: 'de', api_key: SERPAPI_KEY },
|
||||
timeout: 15000
|
||||
});
|
||||
|
||||
@@ -55,73 +55,53 @@ async function getGoogleBathroomPhotosViaApi(hotelName, hotelAddress) {
|
||||
return [];
|
||||
}
|
||||
|
||||
console.log(`📸 [SerpApi] data_id gefunden (${dataId}). Lade Fotoliste...`);
|
||||
|
||||
// B: Fotos & Kategorien abrufen
|
||||
const photosRes = await axios.get('https://serpapi.com/search.json', {
|
||||
params: {
|
||||
engine: 'google_maps_photos',
|
||||
data_id: dataId,
|
||||
hl: 'de',
|
||||
api_key: SERPAPI_KEY
|
||||
},
|
||||
params: { engine: 'google_maps_photos', data_id: dataId, hl: 'de', api_key: SERPAPI_KEY },
|
||||
timeout: 15000
|
||||
});
|
||||
|
||||
const categories = photosRes.data.categories || [];
|
||||
// Flexible Erkennung für Bad-, Zimmer- und Innen-Tags
|
||||
const strictBathKeywords = ['badezimmer', 'bad', 'bathroom', 'wc', 'toilet', 'washroom', 'restroom'];
|
||||
const bathCat = categories.find(c => {
|
||||
const title = (c.title || '').toLowerCase();
|
||||
return title.includes('bad') || title.includes('zimmer') || title.includes('room') || title.includes('innen') || title.includes('gast');
|
||||
const title = (c.title || '').toLowerCase().trim();
|
||||
return strictBathKeywords.some(kw => title.includes(kw));
|
||||
});
|
||||
|
||||
let photosList = photosRes.data.photos || [];
|
||||
|
||||
// Falls Kategorie vorhanden, gezielt Unterkategorie laden
|
||||
let photosList = [];
|
||||
if (bathCat?.id) {
|
||||
console.log(`🚽 [SerpApi] Kategorie gefunden: "${bathCat.title}"`);
|
||||
try {
|
||||
const catRes = await axios.get('https://serpapi.com/search.json', {
|
||||
params: {
|
||||
engine: 'google_maps_photos',
|
||||
data_id: dataId,
|
||||
category_id: bathCat.id,
|
||||
hl: 'de',
|
||||
api_key: SERPAPI_KEY
|
||||
},
|
||||
timeout: 15000
|
||||
});
|
||||
if (catRes.data.photos && catRes.data.photos.length > 0) {
|
||||
photosList = catRes.data.photos;
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('⚠️ [SerpApi] Fallback auf allgemeine Fotos aktiv.');
|
||||
}
|
||||
console.log(`🚽 [SerpApi] Bad-Kategorie gefunden: "${bathCat.title}"`);
|
||||
const catRes = await axios.get('https://serpapi.com/search.json', {
|
||||
params: { engine: 'google_maps_photos', data_id: dataId, category_id: bathCat.id, hl: 'de', api_key: SERPAPI_KEY },
|
||||
timeout: 15000
|
||||
});
|
||||
if (catRes.data.photos?.length > 0) photosList = catRes.data.photos;
|
||||
} else {
|
||||
console.log('ℹ️ [SerpApi] Kein separater Bad-Ordner auf Google Maps vorhanden.');
|
||||
return [];
|
||||
}
|
||||
|
||||
// URLs bereinigen und auf HD-Auflösung setzen
|
||||
const imageUrls = photosList
|
||||
return photosList
|
||||
.map(p => p.image || p.thumbnail)
|
||||
.filter(Boolean)
|
||||
.map(url => url.replace(/=w\d+-h\d+[^=]*/, '=w1200-h800-k-no'))
|
||||
.slice(0, 12);
|
||||
|
||||
console.log(`✅ [SerpApi] ${imageUrls.length} Gästefotos erfolgreich geladen.`);
|
||||
return imageUrls;
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ [SerpApi] Fehler:', error.response?.data?.error || error.message);
|
||||
console.error('❌ [SerpApi] Fehler beim Fotoabruf:', error.message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Worker Definition
|
||||
// ==========================================
|
||||
// BULLMQ WORKER: SCRAPING & REVIEWS
|
||||
// ==========================================
|
||||
const worker = new Worker('scrape-jobs', async (job) => {
|
||||
const { url, webhookUrl, appWebhookUrl, hotelId } = job.data;
|
||||
const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http'));
|
||||
console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews + Bad-Fotos)' : 'PREIS-SCAN (Schnell)'} für: ${url}`);
|
||||
|
||||
// Alle Variablen fest vorinitialisieren
|
||||
console.log(`\n==================================================`);
|
||||
console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews + Fotos)' : 'PREIS-SCAN (Schnell)'} für: ${url}`);
|
||||
console.log(`==================================================`);
|
||||
|
||||
let scrapedPrice = null;
|
||||
let hasGenius = false;
|
||||
let roomSizeM2 = null;
|
||||
@@ -132,21 +112,37 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
||||
let browser = null;
|
||||
|
||||
try {
|
||||
browser = await chromium.launch({ headless: true });
|
||||
// 1. Browser mit Anti-Bot & Stealth-Optionen starten
|
||||
browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--disable-infobars'
|
||||
]
|
||||
});
|
||||
|
||||
const context = await browser.newContext({
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 900 },
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1366, height: 900 },
|
||||
locale: 'de-DE',
|
||||
timezoneId: 'Europe/Berlin'
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
|
||||
// Verhindert Erkennung über navigator.webdriver
|
||||
await page.addInitScript(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
});
|
||||
|
||||
let targetUrl = url;
|
||||
if (isReviewScan && !targetUrl.includes('#tab-reviews')) {
|
||||
targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews';
|
||||
}
|
||||
|
||||
await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 35000 });
|
||||
await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 40000 });
|
||||
await page.waitForTimeout(2500);
|
||||
|
||||
// DATUM AUS URL PARSEN
|
||||
@@ -156,7 +152,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
||||
domData.checkoutDate = parsedUrl.searchParams.get('checkout');
|
||||
} catch (e) {}
|
||||
|
||||
// 1. JSON-LD STAMMDATEN
|
||||
// 2. JSON-LD STAMMDATEN EXTRAHIEREN
|
||||
hotelDetails = await page.evaluate(() => {
|
||||
try {
|
||||
const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
|
||||
@@ -178,7 +174,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
||||
return { name: null, rating: null, reviewCount: null, address: null, image: null };
|
||||
});
|
||||
|
||||
// 2. DOM DATEN
|
||||
// 3. DOM-DATEN (Nächte, Personen, Zimmergröße)
|
||||
const extraDom = await page.evaluate(() => {
|
||||
try {
|
||||
const bodyText = document.body ? (document.body.innerText || '') : '';
|
||||
@@ -214,7 +210,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
||||
return null;
|
||||
});
|
||||
|
||||
// GENIUS BADGE ERKENNUNG
|
||||
// GENIUS BADGE PRÜFUNG
|
||||
hasGenius = await page.evaluate(() => {
|
||||
const geniusSelectors = [
|
||||
'.m-badge__genius',
|
||||
@@ -231,44 +227,175 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
||||
console.log(`[Job ${job.id}] 💙 Genius-Badge erkannt (-10% Rabatt gilt)!`);
|
||||
}
|
||||
|
||||
// 3. KOMMENTARE EXTRAHIEREN
|
||||
// 4. REVIEWS EXTRAHIEREN (STICKY-HEADER AKTIVIERUNG & ECHTER MAUSKLICK)
|
||||
if (isReviewScan) {
|
||||
console.log(`[Job ${job.id}] Starte Review-Extraction mit Pagination...`);
|
||||
console.log(`[Job ${job.id}] 🚀 Starte gezielte Review-Extraction für "${hotelDetails.name || 'Hotel'}"...`);
|
||||
|
||||
// A. Cookie-Banner und störende Overlays schließen
|
||||
try {
|
||||
const reviewTab = await page.$('a[href*="#tab-reviews"], #js--hp-gallery-scorecard a, [data-testid="review-score-link"]');
|
||||
if (reviewTab) {
|
||||
await reviewTab.click();
|
||||
await page.waitForTimeout(2000);
|
||||
const dismissSelectors = [
|
||||
'#onetrust-accept-btn-handler',
|
||||
'button#onetrust-reject-all-handler',
|
||||
'button[data-testid="selection-modal-close-btn"]',
|
||||
'button[aria-label="Schließen"]',
|
||||
'button[aria-label="Close"]'
|
||||
];
|
||||
for (const btnSel of dismissSelectors) {
|
||||
const btn = await page.$(btnSel);
|
||||
if (btn) {
|
||||
await btn.click().catch(() => {});
|
||||
await page.waitForTimeout(400);
|
||||
}
|
||||
}
|
||||
} catch (e) {}
|
||||
|
||||
await page.evaluate(() => window.scrollBy(0, 1000));
|
||||
await page.waitForTimeout(2000);
|
||||
// B. 500px nach unten scrollen, damit Bookings Sticky-Header (#reviews-tab-trigger) überhaupt sichtbar & aktiv wird
|
||||
await page.evaluate(() => window.scrollBy(0, 550));
|
||||
await page.waitForTimeout(1500);
|
||||
|
||||
// C. Klick per echtem Maus-Event auf den Trigger ausführen
|
||||
let clicked = false;
|
||||
try {
|
||||
// Liste möglicher Klick-Ziele (Header-Tab, Scorecard-Badge oder Text "Gästebewertungen")
|
||||
const triggerLocators = [
|
||||
page.locator('#reviews-tab-trigger').first(),
|
||||
page.locator('[data-testid="Property-Header-Nav-Tab-Trigger-reviews"]').first(),
|
||||
page.locator('a:has-text("Gästebewertungen")').first(),
|
||||
page.locator('button:has-text("Gästebewertungen")').first(),
|
||||
page.locator('[data-testid="review-score-right-component"]').first(),
|
||||
page.locator('[data-testid="review-score-component"]').first()
|
||||
];
|
||||
|
||||
for (const loc of triggerLocators) {
|
||||
if (await loc.count() > 0 && await loc.isVisible()) {
|
||||
console.log(`[Job ${job.id}] 👉 Sichtbaren Review-Trigger gefunden. Führe Klick aus...`);
|
||||
|
||||
// Reale Maus-Koordinaten ermitteln & klicken
|
||||
const box = await loc.boundingBox();
|
||||
if (box) {
|
||||
await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2);
|
||||
clicked = true;
|
||||
await page.waitForTimeout(3500);
|
||||
break;
|
||||
} else {
|
||||
await loc.click({ force: true }).catch(() => {});
|
||||
clicked = true;
|
||||
await page.waitForTimeout(3500);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: JS-Klick direkt auf das HTML-Element mit Text "Gästebewertungen"
|
||||
if (!clicked) {
|
||||
console.log(`[Job ${job.id}] ⚠️ Fallback: Klick per DOM-Suche nach 'Gästebewertungen'...`);
|
||||
clicked = await page.evaluate(() => {
|
||||
const el = Array.from(document.querySelectorAll('a, button, span, div'))
|
||||
.find(e => e.innerText && e.innerText.includes('Gästebewertungen') && (e.id === 'reviews-tab-trigger' || e.closest('#reviews-tab-trigger') || e.getAttribute('data-testid')?.includes('reviews')));
|
||||
if (el) {
|
||||
const clickTarget = el.closest('a') || el.closest('button') || el;
|
||||
clickTarget.click();
|
||||
clickTarget.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}).catch(() => false);
|
||||
|
||||
if (clicked) await page.waitForTimeout(3500);
|
||||
}
|
||||
|
||||
// Screenshot zur visuellen Kontrolle aktualisieren
|
||||
try {
|
||||
await page.screenshot({ path: 'src/review-debug.png', fullPage: false });
|
||||
console.log(`[Job ${job.id}] 📸 Screenshot aktualisiert unter: scraper-service/src/review-debug.png`);
|
||||
} catch (e) {}
|
||||
|
||||
} catch (e) {
|
||||
console.log(`[Job ${job.id}] ⚠️ Fehler beim Klick auf Review-Trigger:`, e.message);
|
||||
}
|
||||
|
||||
// Warten bis Review-Karten im Sliding Panel oder DOM sichtbar sind
|
||||
try {
|
||||
await page.waitForSelector(
|
||||
'[data-testid="review-card"], [data-testid="review-cards"], #hp-reviews-sliding, div[data-component="hotel/reviews/review-card"], li.review_item',
|
||||
{ timeout: 8000 }
|
||||
);
|
||||
console.log(`[Job ${job.id}] 👁️ Review-Bereich erfolgreich gerendert.`);
|
||||
} catch (e) {
|
||||
console.log(`[Job ${job.id}] ⚠️ Wartezeit abgelaufen – starte Auslesen aus dem aktuellen DOM...`);
|
||||
}
|
||||
|
||||
let pageCount = 0;
|
||||
const seenKeys = new Set();
|
||||
|
||||
while (extractedReviews.length < 80 && pageCount < 15) {
|
||||
const pageReviews = await page.evaluate(({ westernCountries }) => {
|
||||
const cards = Array.from(document.querySelectorAll('[data-testid="review-card"], [role="group"][aria-label*="Bewertung"], li.review_list_new_item_block, .review_item'));
|
||||
const results = [];
|
||||
const currentYear = 2026;
|
||||
for (const card of cards) {
|
||||
const cardText = card ? (card.innerText || '') : '';
|
||||
if (!cardText) continue;
|
||||
const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase()));
|
||||
if (!isWestern) continue;
|
||||
while (pageCount < 30) {
|
||||
pageCount++;
|
||||
console.log(`[Job ${job.id}] 📄 Scanne Bewertungsseite ${pageCount} (Bisher gesammelt: ${extractedReviews.length})...`);
|
||||
|
||||
const evalResult = await page.evaluate(({ westernCountries }) => {
|
||||
let cards = Array.from(document.querySelectorAll([
|
||||
'[data-testid="review-card"]',
|
||||
'[data-testid="review-cards"] > li',
|
||||
'[data-testid="review-cards"] > div',
|
||||
'div[data-testid*="review-card"]',
|
||||
'li[data-testid*="review-card"]',
|
||||
'li.review_list_new_item_block',
|
||||
'div.c-review-block__item',
|
||||
'div[itemprop="review"]',
|
||||
'li.review_item'
|
||||
].join(',')));
|
||||
|
||||
if (cards.length === 0) {
|
||||
const container = document.querySelector('#hp-reviews-sliding, #reviewCardsSection, [data-testid="review-cards"], #blockdisplay4');
|
||||
if (container) {
|
||||
cards = Array.from(container.querySelectorAll('li, div')).filter(el => (el.innerText || '').length > 40);
|
||||
}
|
||||
}
|
||||
|
||||
// Filterboxen und Scorecard-Zusammenfassungen explizit ausschließen
|
||||
cards = cards.filter(el => {
|
||||
const text = el.innerText || '';
|
||||
return text.length > 25 &&
|
||||
!text.includes('Wählen Sie Themen') &&
|
||||
!text.includes('Kategorien:') &&
|
||||
!el.classList.contains('review_score_breakdown');
|
||||
});
|
||||
|
||||
const results = [];
|
||||
const debugLogs = [];
|
||||
const currentYear = 2026;
|
||||
|
||||
for (let idx = 0; idx < cards.length; idx++) {
|
||||
const card = cards[idx];
|
||||
const cardText = card.innerText || '';
|
||||
if (!cardText.trim()) continue;
|
||||
|
||||
// 1. Herkunftsland des Gastes ermitteln
|
||||
const countryEl = card.querySelector('[data-testid="review-country"], .bui-avatar-or-image__title, .user_country, .c-review__country');
|
||||
const guestCountryText = countryEl ? countryEl.innerText : cardText;
|
||||
|
||||
const isWestern = westernCountries.some(c => guestCountryText.toLowerCase().includes(c.toLowerCase()));
|
||||
if (!isWestern) {
|
||||
const preview = guestCountryText.substring(0, 50).replace(/\n/g, ' ');
|
||||
debugLogs.push(`Karte #${idx + 1}: ❌ Land nicht in Liste ("${preview}")`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 2. Datum / Jahr prüfen (letzte 2 Jahre)
|
||||
const yearMatch = cardText.match(/\b(20\d{2})\b/);
|
||||
if (yearMatch) {
|
||||
const reviewYear = parseInt(yearMatch[1], 10);
|
||||
if (currentYear - reviewYear > 2) continue;
|
||||
if (currentYear - reviewYear > 2) {
|
||||
debugLogs.push(`Karte #${idx + 1}: ❌ Zu alt (${reviewYear})`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const titleEl = card.querySelector('[data-testid="review-title"], h4');
|
||||
const posEl = card.querySelector('[data-testid="review-positive-text"], .review_pos');
|
||||
const negEl = card.querySelector('[data-testid="review-negative-text"], .review_neg');
|
||||
const dateEl = card.querySelector('[data-testid="review-date"]');
|
||||
// 3. Texte extrahieren
|
||||
const titleEl = card.querySelector('[data-testid="review-title"], .c-review__title, h4, .review_title');
|
||||
const posEl = card.querySelector('[data-testid="review-positive-text"], .c-review__row--positive, .review_pos');
|
||||
const negEl = card.querySelector('[data-testid="review-negative-text"], .c-review__row--negative, .review_neg');
|
||||
const dateEl = card.querySelector('[data-testid="review-date"], .c-review__date, .review_item_date');
|
||||
|
||||
const title = titleEl ? titleEl.innerText.trim() : '';
|
||||
const positive = posEl ? posEl.innerText.trim() : '';
|
||||
@@ -276,16 +403,30 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
||||
const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell');
|
||||
|
||||
let fallbackText = '';
|
||||
if (!positive && !negative) fallbackText = cardText.replace(title, '').substring(0, 300).trim();
|
||||
if (!positive && !negative) {
|
||||
fallbackText = cardText.replace(title, '').substring(0, 300).trim();
|
||||
}
|
||||
|
||||
if (title || positive || negative || fallbackText) {
|
||||
results.push({ title, positive: positive || fallbackText, negative, date: dateText });
|
||||
debugLogs.push(`Karte #${idx + 1}: ✅ Akzeptiert ("${title || (positive ? positive.substring(0, 25) : 'Review')}")`);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}, { westernCountries: WESTERN_COUNTRIES }).catch(() => []);
|
||||
|
||||
for (const rev of pageReviews) {
|
||||
return {
|
||||
totalCards: cards.length,
|
||||
results,
|
||||
debugLogs: debugLogs.slice(0, 8)
|
||||
};
|
||||
}, { westernCountries: WESTERN_COUNTRIES });
|
||||
|
||||
console.log(`[Job ${job.id}] 🔍 Echte Gästekarten gefunden: ${evalResult.totalCards}`);
|
||||
if (evalResult.debugLogs) {
|
||||
evalResult.debugLogs.forEach(log => console.log(`[Job ${job.id}] └─ ${log}`));
|
||||
}
|
||||
|
||||
// Deduplizierung
|
||||
for (const rev of evalResult.results) {
|
||||
const key = `${rev.title}-${rev.positive}-${rev.date}`;
|
||||
if (!seenKeys.has(key) && extractedReviews.length < 80) {
|
||||
seenKeys.add(key);
|
||||
@@ -295,26 +436,32 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
||||
|
||||
if (extractedReviews.length >= 80) break;
|
||||
|
||||
// Weiterblättern im Modal oder der Hauptseite
|
||||
try {
|
||||
const nextBtn = await page.$('#reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"]');
|
||||
const nextBtn = await page.$(
|
||||
'#hp-reviews-sliding button[aria-label*="Nächste"], #hp-reviews-sliding button[aria-label*="Next"], #reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"], a.paging-next, li.bui-pagination__next-arrow a'
|
||||
);
|
||||
if (nextBtn) {
|
||||
const isDisabled = await nextBtn.getAttribute('disabled');
|
||||
if (isDisabled !== null) break;
|
||||
await nextBtn.click();
|
||||
await page.waitForTimeout(2000);
|
||||
pageCount++;
|
||||
if (isDisabled === null) {
|
||||
await nextBtn.click({ force: true });
|
||||
await page.waitForTimeout(2500);
|
||||
} else break;
|
||||
} else break;
|
||||
} catch (e) { break; }
|
||||
} catch (e) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
console.log(`[Job ${job.id}] ${extractedReviews.length} Kommentare extrahiert.`);
|
||||
|
||||
console.log(`[Job ${job.id}] ✅ Insgesamt ${extractedReviews.length} passende Kommentare extrahiert.`);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error(`[Job ${job.id}] Fehler beim Scraping:`, error.message);
|
||||
console.error(`[Job ${job.id}] ❌ Fehler beim Scraping:`, error.message);
|
||||
} finally {
|
||||
if (browser) await browser.close();
|
||||
|
||||
// 4. Google Maps Fotos nur bei gezieltem KI-Review-Scan abrufen (spart SerpApi-Credits)
|
||||
// 5. GOOGLE MAPS FOTOS NUR BEI KI-REVIEW SCAN ABRUFEN
|
||||
if (isReviewScan && hotelDetails.name) {
|
||||
try {
|
||||
bathroomImages = await getGoogleBathroomPhotosViaApi(hotelDetails.name, hotelDetails.address);
|
||||
@@ -324,7 +471,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Preishistorie und Bad-Bilder an Travel-App Webhook senden
|
||||
// 6. PREISHISTORIE + STAMMDATEN AN NEXT.JS WEBHOOK SENDEN
|
||||
if (appWebhookUrl) {
|
||||
try {
|
||||
await axios.post(appWebhookUrl, {
|
||||
@@ -342,13 +489,13 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
||||
persons: domData.dPersons || 2,
|
||||
bathroomImages: Array.isArray(bathroomImages) && bathroomImages.length > 0 ? bathroomImages : undefined
|
||||
});
|
||||
console.log(`[Job ${job.id}] ✅ Preishistorie in Postgres gespeichert!`);
|
||||
console.log(`[Job ${job.id}] ✅ Preishistorie & Stammdaten in Postgres gespeichert!`);
|
||||
} catch (e) {
|
||||
console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie:`, e.message);
|
||||
console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie-Webhook:`, e.message);
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Reviews an n8n senden
|
||||
// 7. REVIEWS AN N8N WEBHOOK SENDEN
|
||||
if (isReviewScan && extractedReviews.length > 0) {
|
||||
try {
|
||||
await axios.post(webhookUrl, {
|
||||
@@ -359,7 +506,9 @@ const worker = new Worker('scrape-jobs', async (job) => {
|
||||
timestamp: new Date().toISOString()
|
||||
});
|
||||
console.log(`[Job ${job.id}] 🚀 ${extractedReviews.length} Reviews an n8n gesendet!`);
|
||||
} catch (e) {}
|
||||
} catch (e) {
|
||||
console.error(`[Job ${job.id}] ❌ Fehler beim Senden an n8n:`, e.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}, { ...REDIS_CONFIG, concurrency: 2 });
|
||||
|
||||
BIN
src/review-debug.png
Normal file
BIN
src/review-debug.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 342 KiB |
Reference in New Issue
Block a user