This commit is contained in:
2026-08-16 10:43:16 +00:00
parent e5511cc4ff
commit 2859f2c503
3 changed files with 128 additions and 14 deletions

1
.env Normal file
View File

@@ -0,0 +1 @@
SERPAPI_KEY=9e30106018e6a8663078fb9ead5249b53feb90ed6291b84ee3d52cc90dc3d840

View File

@@ -1,5 +1,4 @@
version: '3.8' version: '3.8'
services: services:
redis: redis:
image: redis:alpine image: redis:alpine
@@ -17,5 +16,10 @@ services:
environment: environment:
- REDIS_HOST=redis - REDIS_HOST=redis
- REDIS_PORT=6379 - REDIS_PORT=6379
- SERPAPI_KEY=${SERPAPI_KEY}
env_file:
- .env
volumes:
- ./src:/app/src
depends_on: depends_on:
- redis - redis

View File

@@ -9,6 +9,8 @@ const REDIS_CONFIG = {
} }
}; };
const SERPAPI_KEY = process.env.SERPAPI_KEY || null;
const WESTERN_COUNTRIES = [ const WESTERN_COUNTRIES = [
'Deutschland', 'Schweiz', 'Österreich', 'Vereinigte Staaten', 'USA', 'Deutschland', 'Schweiz', 'Österreich', 'Vereinigte Staaten', 'USA',
'Vereinigtes Königreich', 'Großbritannien', 'Niederlande', 'Frankreich', 'Vereinigtes Königreich', 'Großbritannien', 'Niederlande', 'Frankreich',
@@ -16,18 +18,117 @@ const WESTERN_COUNTRIES = [
'Norwegen', 'Dänemark', 'Finnland', 'Irland', 'Luxemburg', 'Indien' 'Norwegen', 'Dänemark', 'Finnland', 'Irland', 'Luxemburg', 'Indien'
]; ];
// 1. Google Maps Fotos via SerpApi abrufen (inkl. Adressbereinigung & Fallback)
async function getGoogleBathroomPhotosViaApi(hotelName, hotelAddress) {
if (!SERPAPI_KEY) {
console.log('⚠️ [SerpApi] Kein SERPAPI_KEY in der .env hinterlegt!');
return [];
}
try {
// Booking-Adresse bereinigen: Nur Stadt und Land nutzen für maximale Trefferquote
let cleanLocation = '';
if (hotelAddress) {
const parts = hotelAddress.split(',').map(p => p.trim());
cleanLocation = parts.length > 1 ? parts.slice(-2).join(' ') : hotelAddress;
}
const query = `${hotelName} ${cleanLocation}`.trim();
console.log(`🔍 [SerpApi] Suche Google Maps Place für: "${query}"...`);
// A: Google Maps Ort suchen
const searchRes = await axios.get('https://serpapi.com/search.json', {
params: {
engine: 'google_maps',
q: query,
hl: 'de',
api_key: SERPAPI_KEY
},
timeout: 15000
});
const placeResults = searchRes.data.place_results;
const localResults = searchRes.data.local_results;
const dataId = placeResults?.data_id || (localResults && localResults[0]?.data_id) || searchRes.data.search_metadata?.google_maps_data_id;
if (!dataId) {
console.log(`⚠️ [SerpApi] Keine Google Maps data_id für "${query}" gefunden.`);
return [];
}
console.log(`📸 [SerpApi] data_id gefunden (${dataId}). Lade Fotoliste...`);
// B: Fotos & Kategorien abrufen
const photosRes = await axios.get('https://serpapi.com/search.json', {
params: {
engine: 'google_maps_photos',
data_id: dataId,
hl: 'de',
api_key: SERPAPI_KEY
},
timeout: 15000
});
const categories = photosRes.data.categories || [];
// Flexible Erkennung für Bad-, Zimmer- und Innen-Tags
const bathCat = categories.find(c => {
const title = (c.title || '').toLowerCase();
return title.includes('bad') || title.includes('zimmer') || title.includes('room') || title.includes('innen') || title.includes('gast');
});
let photosList = photosRes.data.photos || [];
// Falls Kategorie vorhanden, gezielt Unterkategorie laden
if (bathCat?.id) {
console.log(`🚽 [SerpApi] Kategorie gefunden: "${bathCat.title}"`);
try {
const catRes = await axios.get('https://serpapi.com/search.json', {
params: {
engine: 'google_maps_photos',
data_id: dataId,
category_id: bathCat.id,
hl: 'de',
api_key: SERPAPI_KEY
},
timeout: 15000
});
if (catRes.data.photos && catRes.data.photos.length > 0) {
photosList = catRes.data.photos;
}
} catch (e) {
console.log('⚠️ [SerpApi] Fallback auf allgemeine Fotos aktiv.');
}
}
// URLs bereinigen und auf HD-Auflösung setzen
const imageUrls = photosList
.map(p => p.image || p.thumbnail)
.filter(Boolean)
.map(url => url.replace(/=w\d+-h\d+[^=]*/, '=w1200-h800-k-no'))
.slice(0, 12);
console.log(`✅ [SerpApi] ${imageUrls.length} Gästefotos erfolgreich geladen.`);
return imageUrls;
} catch (error) {
console.error('❌ [SerpApi] Fehler:', error.response?.data?.error || error.message);
return [];
}
}
// 2. Worker Definition
const worker = new Worker('scrape-jobs', async (job) => { const worker = new Worker('scrape-jobs', async (job) => {
const { url, webhookUrl, appWebhookUrl, hotelId } = job.data; const { url, webhookUrl, appWebhookUrl, hotelId } = job.data;
const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http')); const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http'));
console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews + Bad-Fotos)' : 'PREIS-SCAN (Schnell)'} für: ${url}`);
console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews)' : 'PREIS-SCAN (Schnell)'} für: ${url}`); // Alle Variablen fest vorinitialisieren
let scrapedPrice = null; let scrapedPrice = null;
let hasGenius = false; let hasGenius = false;
let roomSizeM2 = null; let roomSizeM2 = null;
let hotelDetails = { name: null, rating: null, reviewCount: null, address: null, image: null }; let hotelDetails = { name: null, rating: null, reviewCount: null, address: null, image: null };
let domData = { dNights: 1, dPersons: 2, roomSize: null, checkinDate: null, checkoutDate: null }; let domData = { dNights: 1, dPersons: 2, roomSize: null, checkinDate: null, checkoutDate: null };
let extractedReviews = []; let extractedReviews = [];
let bathroomImages = [];
let browser = null; let browser = null;
try { try {
@@ -38,8 +139,8 @@ const worker = new Worker('scrape-jobs', async (job) => {
locale: 'de-DE', locale: 'de-DE',
timezoneId: 'Europe/Berlin' timezoneId: 'Europe/Berlin'
}); });
const page = await context.newPage(); const page = await context.newPage();
let targetUrl = url; let targetUrl = url;
if (isReviewScan && !targetUrl.includes('#tab-reviews')) { if (isReviewScan && !targetUrl.includes('#tab-reviews')) {
targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews'; targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews';
@@ -53,7 +154,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
const parsedUrl = new URL(url); const parsedUrl = new URL(url);
domData.checkinDate = parsedUrl.searchParams.get('checkin'); domData.checkinDate = parsedUrl.searchParams.get('checkin');
domData.checkoutDate = parsedUrl.searchParams.get('checkout'); domData.checkoutDate = parsedUrl.searchParams.get('checkout');
} catch(e) {} } catch (e) {}
// 1. JSON-LD STAMMDATEN // 1. JSON-LD STAMMDATEN
hotelDetails = await page.evaluate(() => { hotelDetails = await page.evaluate(() => {
@@ -113,7 +214,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
return null; return null;
}); });
// GENIUS BADGE ERKENNUNG (NEU) // GENIUS BADGE ERKENNUNG
hasGenius = await page.evaluate(() => { hasGenius = await page.evaluate(() => {
const geniusSelectors = [ const geniusSelectors = [
'.m-badge__genius', '.m-badge__genius',
@@ -152,11 +253,9 @@ const worker = new Worker('scrape-jobs', async (job) => {
const cards = Array.from(document.querySelectorAll('[data-testid="review-card"], [role="group"][aria-label*="Bewertung"], li.review_list_new_item_block, .review_item')); const cards = Array.from(document.querySelectorAll('[data-testid="review-card"], [role="group"][aria-label*="Bewertung"], li.review_list_new_item_block, .review_item'));
const results = []; const results = [];
const currentYear = 2026; const currentYear = 2026;
for (const card of cards) { for (const card of cards) {
const cardText = card ? (card.innerText || '') : ''; const cardText = card ? (card.innerText || '') : '';
if (!cardText) continue; if (!cardText) continue;
const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase())); const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase()));
if (!isWestern) continue; if (!isWestern) continue;
@@ -215,7 +314,17 @@ const worker = new Worker('scrape-jobs', async (job) => {
} finally { } finally {
if (browser) await browser.close(); if (browser) await browser.close();
// 1. PREISHISTORIE + DATUM SPEICHERN // 4. Google Maps Fotos immer abrufen, sobald ein Hotelname bekannt ist
if (hotelDetails.name) {
try {
bathroomImages = await getGoogleBathroomPhotosViaApi(hotelDetails.name, hotelDetails.address);
} catch (err) {
console.error('Foto-Abruf fehlgeschlagen:', err.message);
bathroomImages = [];
}
}
// 5. Preishistorie und Bad-Bilder an Travel-App Webhook senden
if (appWebhookUrl) { if (appWebhookUrl) {
try { try {
await axios.post(appWebhookUrl, { await axios.post(appWebhookUrl, {
@@ -231,6 +340,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
checkoutDate: domData.checkoutDate, checkoutDate: domData.checkoutDate,
nights: domData.dNights || 1, nights: domData.dNights || 1,
persons: domData.dPersons || 2, persons: domData.dPersons || 2,
bathroomImages: Array.isArray(bathroomImages) && bathroomImages.length > 0 ? bathroomImages : undefined
}); });
console.log(`[Job ${job.id}] ✅ Preishistorie in Postgres gespeichert!`); console.log(`[Job ${job.id}] ✅ Preishistorie in Postgres gespeichert!`);
} catch (e) { } catch (e) {
@@ -238,7 +348,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
} }
} }
// 2. REVIEWS AN N8N SENDEN // 6. Reviews an n8n senden
if (isReviewScan && extractedReviews.length > 0) { if (isReviewScan && extractedReviews.length > 0) {
try { try {
await axios.post(webhookUrl, { await axios.post(webhookUrl, {
@@ -252,7 +362,6 @@ const worker = new Worker('scrape-jobs', async (job) => {
} catch (e) {} } catch (e) {}
} }
} }
}, { ...REDIS_CONFIG, concurrency: 2 }); }, { ...REDIS_CONFIG, concurrency: 2 });
console.log('🚀 Scraper Worker bereit...'); console.log('🚀 Scraper Worker bereit...');