From 2859f2c503d89ea05bd61e11d18a3ad409299839 Mon Sep 17 00:00:00 2001 From: gitea Date: Sun, 16 Aug 2026 10:43:16 +0000 Subject: [PATCH] Bilder --- .env | 1 + docker-compose.yml | 8 ++- src/index.js | 133 +++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 128 insertions(+), 14 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..c77fb6d --- /dev/null +++ b/.env @@ -0,0 +1 @@ +SERPAPI_KEY=9e30106018e6a8663078fb9ead5249b53feb90ed6291b84ee3d52cc90dc3d840 diff --git a/docker-compose.yml b/docker-compose.yml index f1d5454..d93dccf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,4 @@ version: '3.8' - services: redis: image: redis:alpine @@ -17,5 +16,10 @@ services: environment: - REDIS_HOST=redis - REDIS_PORT=6379 + - SERPAPI_KEY=${SERPAPI_KEY} + env_file: + - .env + volumes: + - ./src:/app/src depends_on: - - redis + - redis \ No newline at end of file diff --git a/src/index.js b/src/index.js index b1efb20..0be5fd6 100644 --- a/src/index.js +++ b/src/index.js @@ -9,6 +9,8 @@ const REDIS_CONFIG = { } }; +const SERPAPI_KEY = process.env.SERPAPI_KEY || null; + const WESTERN_COUNTRIES = [ 'Deutschland', 'Schweiz', 'Österreich', 'Vereinigte Staaten', 'USA', 'Vereinigtes Königreich', 'Großbritannien', 'Niederlande', 'Frankreich', @@ -16,18 +18,117 @@ const WESTERN_COUNTRIES = [ 'Norwegen', 'Dänemark', 'Finnland', 'Irland', 'Luxemburg', 'Indien' ]; +// 1. Google Maps Fotos via SerpApi abrufen (inkl. Adressbereinigung & Fallback) +async function getGoogleBathroomPhotosViaApi(hotelName, hotelAddress) { + if (!SERPAPI_KEY) { + console.log('⚠️ [SerpApi] Kein SERPAPI_KEY in der .env hinterlegt!'); + return []; + } + + try { + // Booking-Adresse bereinigen: Nur Stadt und Land nutzen für maximale Trefferquote + let cleanLocation = ''; + if (hotelAddress) { + const parts = hotelAddress.split(',').map(p => p.trim()); + cleanLocation = parts.length > 1 ? parts.slice(-2).join(' ') : hotelAddress; + } + const query = `${hotelName} ${cleanLocation}`.trim(); + console.log(`🔍 [SerpApi] Suche Google Maps Place für: "${query}"...`); + + // A: Google Maps Ort suchen + const searchRes = await axios.get('https://serpapi.com/search.json', { + params: { + engine: 'google_maps', + q: query, + hl: 'de', + api_key: SERPAPI_KEY + }, + timeout: 15000 + }); + + const placeResults = searchRes.data.place_results; + const localResults = searchRes.data.local_results; + const dataId = placeResults?.data_id || (localResults && localResults[0]?.data_id) || searchRes.data.search_metadata?.google_maps_data_id; + + if (!dataId) { + console.log(`⚠️ [SerpApi] Keine Google Maps data_id für "${query}" gefunden.`); + return []; + } + + console.log(`📸 [SerpApi] data_id gefunden (${dataId}). Lade Fotoliste...`); + + // B: Fotos & Kategorien abrufen + const photosRes = await axios.get('https://serpapi.com/search.json', { + params: { + engine: 'google_maps_photos', + data_id: dataId, + hl: 'de', + api_key: SERPAPI_KEY + }, + timeout: 15000 + }); + + const categories = photosRes.data.categories || []; + // Flexible Erkennung für Bad-, Zimmer- und Innen-Tags + const bathCat = categories.find(c => { + const title = (c.title || '').toLowerCase(); + return title.includes('bad') || title.includes('zimmer') || title.includes('room') || title.includes('innen') || title.includes('gast'); + }); + + let photosList = photosRes.data.photos || []; + + // Falls Kategorie vorhanden, gezielt Unterkategorie laden + if (bathCat?.id) { + console.log(`🚽 [SerpApi] Kategorie gefunden: "${bathCat.title}"`); + try { + const catRes = await axios.get('https://serpapi.com/search.json', { + params: { + engine: 'google_maps_photos', + data_id: dataId, + category_id: bathCat.id, + hl: 'de', + api_key: SERPAPI_KEY + }, + timeout: 15000 + }); + if (catRes.data.photos && catRes.data.photos.length > 0) { + photosList = catRes.data.photos; + } + } catch (e) { + console.log('⚠️ [SerpApi] Fallback auf allgemeine Fotos aktiv.'); + } + } + + // URLs bereinigen und auf HD-Auflösung setzen + const imageUrls = photosList + .map(p => p.image || p.thumbnail) + .filter(Boolean) + .map(url => url.replace(/=w\d+-h\d+[^=]*/, '=w1200-h800-k-no')) + .slice(0, 12); + + console.log(`✅ [SerpApi] ${imageUrls.length} Gästefotos erfolgreich geladen.`); + return imageUrls; + + } catch (error) { + console.error('❌ [SerpApi] Fehler:', error.response?.data?.error || error.message); + return []; + } +} + +// 2. Worker Definition const worker = new Worker('scrape-jobs', async (job) => { const { url, webhookUrl, appWebhookUrl, hotelId } = job.data; const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http')); - - console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews)' : 'PREIS-SCAN (Schnell)'} für: ${url}`); - + console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews + Bad-Fotos)' : 'PREIS-SCAN (Schnell)'} für: ${url}`); + + // Alle Variablen fest vorinitialisieren let scrapedPrice = null; let hasGenius = false; let roomSizeM2 = null; let hotelDetails = { name: null, rating: null, reviewCount: null, address: null, image: null }; let domData = { dNights: 1, dPersons: 2, roomSize: null, checkinDate: null, checkoutDate: null }; let extractedReviews = []; + let bathroomImages = []; let browser = null; try { @@ -38,8 +139,8 @@ const worker = new Worker('scrape-jobs', async (job) => { locale: 'de-DE', timezoneId: 'Europe/Berlin' }); - const page = await context.newPage(); + let targetUrl = url; if (isReviewScan && !targetUrl.includes('#tab-reviews')) { targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews'; @@ -53,7 +154,7 @@ const worker = new Worker('scrape-jobs', async (job) => { const parsedUrl = new URL(url); domData.checkinDate = parsedUrl.searchParams.get('checkin'); domData.checkoutDate = parsedUrl.searchParams.get('checkout'); - } catch(e) {} + } catch (e) {} // 1. JSON-LD STAMMDATEN hotelDetails = await page.evaluate(() => { @@ -113,7 +214,7 @@ const worker = new Worker('scrape-jobs', async (job) => { return null; }); - // GENIUS BADGE ERKENNUNG (NEU) + // GENIUS BADGE ERKENNUNG hasGenius = await page.evaluate(() => { const geniusSelectors = [ '.m-badge__genius', @@ -152,11 +253,9 @@ const worker = new Worker('scrape-jobs', async (job) => { const cards = Array.from(document.querySelectorAll('[data-testid="review-card"], [role="group"][aria-label*="Bewertung"], li.review_list_new_item_block, .review_item')); const results = []; const currentYear = 2026; - for (const card of cards) { const cardText = card ? (card.innerText || '') : ''; if (!cardText) continue; - const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase())); if (!isWestern) continue; @@ -215,7 +314,17 @@ const worker = new Worker('scrape-jobs', async (job) => { } finally { if (browser) await browser.close(); - // 1. PREISHISTORIE + DATUM SPEICHERN +// 4. Google Maps Fotos immer abrufen, sobald ein Hotelname bekannt ist + if (hotelDetails.name) { + try { + bathroomImages = await getGoogleBathroomPhotosViaApi(hotelDetails.name, hotelDetails.address); + } catch (err) { + console.error('Foto-Abruf fehlgeschlagen:', err.message); + bathroomImages = []; + } + } + + // 5. Preishistorie und Bad-Bilder an Travel-App Webhook senden if (appWebhookUrl) { try { await axios.post(appWebhookUrl, { @@ -231,6 +340,7 @@ const worker = new Worker('scrape-jobs', async (job) => { checkoutDate: domData.checkoutDate, nights: domData.dNights || 1, persons: domData.dPersons || 2, + bathroomImages: Array.isArray(bathroomImages) && bathroomImages.length > 0 ? bathroomImages : undefined }); console.log(`[Job ${job.id}] ✅ Preishistorie in Postgres gespeichert!`); } catch (e) { @@ -238,7 +348,7 @@ const worker = new Worker('scrape-jobs', async (job) => { } } - // 2. REVIEWS AN N8N SENDEN + // 6. Reviews an n8n senden if (isReviewScan && extractedReviews.length > 0) { try { await axios.post(webhookUrl, { @@ -252,7 +362,6 @@ const worker = new Worker('scrape-jobs', async (job) => { } catch (e) {} } } - }, { ...REDIS_CONFIG, concurrency: 2 }); -console.log('🚀 Scraper Worker bereit...'); +console.log('🚀 Scraper Worker bereit...'); \ No newline at end of file