From e5511cc4ff4cb7e7831a3b58e3c9180242bf9101 Mon Sep 17 00:00:00 2001 From: gitea Date: Sun, 16 Aug 2026 10:09:01 +0000 Subject: [PATCH] Preishistory mit Genius --- copyChat.sh | 80 +++++++++ llm_context.txt | 436 ++++++++++++++++++++++++++++++++++++++++++++++++ src/index.js | 326 ++++++++++++++++++------------------ 3 files changed, 676 insertions(+), 166 deletions(-) create mode 100755 copyChat.sh create mode 100644 llm_context.txt diff --git a/copyChat.sh b/copyChat.sh new file mode 100755 index 0000000..bada8fa --- /dev/null +++ b/copyChat.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +OUTPUT_FILE="${1:-llm_context.txt}" +ROOT_DIR="${2:-.}" + +# Absolute Pfade auflösen +OUTPUT_PATH="$(cd "$(dirname "$OUTPUT_FILE")" 2>/dev/null && pwd)/$(basename "$OUTPUT_FILE")" + +cd "$ROOT_DIR" || exit 1 + +# Ausgabedatei initialisieren / leeren +> "$OUTPUT_PATH" + +echo "========================================" >> "$OUTPUT_PATH" +echo "PROJEKTSTRUKTUR" >> "$OUTPUT_PATH" +echo "========================================" >> "$OUTPUT_PATH" + +# Prüfen, ob Git verfügbar und das Verzeichnis ein Git-Repo ist +IS_GIT_REPO=false +if command -v git &> /dev/null && git rev-parse --is-inside-work-tree &> /dev/null; then + IS_GIT_REPO=true +fi + +if [ "$IS_GIT_REPO" = true ]; then + echo "(Ermittelt via Git unter Berücksichtigung von .gitignore)" >> "$OUTPUT_PATH" + # Zeigt alle getrackten + ungetrackten Dateien an, ignoriert .gitignore-Treffer + git ls-files -co --exclude-standard >> "$OUTPUT_PATH" +else + if command -v tree &> /dev/null; then + tree -I 'node_modules|.git|venv|.venv|__pycache__|dist|build|.next|target' >> "$OUTPUT_PATH" + else + find . -maxdepth 3 -not -path '*/.*' -not -path '*/node_modules*' -not -path '*/venv*' >> "$OUTPUT_PATH" + fi +fi + +echo -e "\n========================================" >> "$OUTPUT_PATH" +echo "DATEIINHALTE" >> "$OUTPUT_PATH" +echo "========================================" >> "$OUTPUT_PATH" + +# Dateiliste ermitteln +get_file_list() { + if [ "$IS_GIT_REPO" = true ]; then + # -c = cached/tracked, -o = others/untracked, --exclude-standard = .gitignore beachten + git ls-files -co --exclude-standard + else + find . -type f \ + -not -path '*/.git/*' \ + -not -path '*/node_modules/*' \ + -not -path '*/venv/*' \ + -not -path '*/.venv/*' \ + -not -path '*/__pycache__/*' \ + -not -path '*/dist/*' \ + -not -path '*/build/*' \ + -not -path '*/.next/*' + fi +} + +# Dateien verarbeiten +get_file_list | sort | while IFS= read -r file; do + # Überspringe leere Pfade, die Zieldatei selbst oder typische Binär-Endungen + [ -z "$file" ] && continue + [ "$file" = "$OUTPUT_FILE" ] && continue + [ "$(realpath "$file" 2>/dev/null)" = "$OUTPUT_PATH" ] && continue + + case "$file" in + *.png|*.jpg|*.jpeg|*.gif|*.ico|*.pdf|*.zip|*.tar.gz|*.lock|package-lock.json|yarn.lock) + continue + ;; + esac + + # Prüfen, ob die Datei existiert und lesbarer Text ist + if [ -f "$file" ] && (grep -Iq . "$file" 2>/dev/null || [ ! -s "$file" ]); then + echo -e "\n--- DATEI: $file ---" >> "$OUTPUT_PATH" + echo '```' >> "$OUTPUT_PATH" + cat "$file" >> "$OUTPUT_PATH" + echo -e "\n"'```' >> "$OUTPUT_PATH" + fi +done + +echo "Fertig! Projektkontext liegt in: $OUTPUT_PATH" \ No newline at end of file diff --git a/llm_context.txt b/llm_context.txt new file mode 100644 index 0000000..c9ab856 --- /dev/null +++ b/llm_context.txt @@ -0,0 +1,436 @@ +======================================== +PROJEKTSTRUKTUR +======================================== +(Ermittelt via Git unter Berücksichtigung von .gitignore) +copyChat.sh +llm_context.txt +Dockerfile +README.md +docker-compose.yml +package.json +src/index.js + +======================================== +DATEIINHALTE +======================================== + +--- DATEI: Dockerfile --- +``` +FROM mcr.microsoft.com/playwright:v1.45.0-jammy + +WORKDIR /app + +# Package-Dateien kopieren und installieren +COPY package*.json ./ +RUN npm install + +# Sourcecode kopieren +COPY . . + +EXPOSE 3000 + +CMD ["node", "src/index.js"] + +``` + +--- DATEI: README.md --- +``` + +``` + +--- DATEI: copyChat.sh --- +``` +#!/usr/bin/env bash + +OUTPUT_FILE="${1:-llm_context.txt}" +ROOT_DIR="${2:-.}" + +# Absolute Pfade auflösen +OUTPUT_PATH="$(cd "$(dirname "$OUTPUT_FILE")" 2>/dev/null && pwd)/$(basename "$OUTPUT_FILE")" + +cd "$ROOT_DIR" || exit 1 + +# Ausgabedatei initialisieren / leeren +> "$OUTPUT_PATH" + +echo "========================================" >> "$OUTPUT_PATH" +echo "PROJEKTSTRUKTUR" >> "$OUTPUT_PATH" +echo "========================================" >> "$OUTPUT_PATH" + +# Prüfen, ob Git verfügbar und das Verzeichnis ein Git-Repo ist +IS_GIT_REPO=false +if command -v git &> /dev/null && git rev-parse --is-inside-work-tree &> /dev/null; then + IS_GIT_REPO=true +fi + +if [ "$IS_GIT_REPO" = true ]; then + echo "(Ermittelt via Git unter Berücksichtigung von .gitignore)" >> "$OUTPUT_PATH" + # Zeigt alle getrackten + ungetrackten Dateien an, ignoriert .gitignore-Treffer + git ls-files -co --exclude-standard >> "$OUTPUT_PATH" +else + if command -v tree &> /dev/null; then + tree -I 'node_modules|.git|venv|.venv|__pycache__|dist|build|.next|target' >> "$OUTPUT_PATH" + else + find . -maxdepth 3 -not -path '*/.*' -not -path '*/node_modules*' -not -path '*/venv*' >> "$OUTPUT_PATH" + fi +fi + +echo -e "\n========================================" >> "$OUTPUT_PATH" +echo "DATEIINHALTE" >> "$OUTPUT_PATH" +echo "========================================" >> "$OUTPUT_PATH" + +# Dateiliste ermitteln +get_file_list() { + if [ "$IS_GIT_REPO" = true ]; then + # -c = cached/tracked, -o = others/untracked, --exclude-standard = .gitignore beachten + git ls-files -co --exclude-standard + else + find . -type f \ + -not -path '*/.git/*' \ + -not -path '*/node_modules/*' \ + -not -path '*/venv/*' \ + -not -path '*/.venv/*' \ + -not -path '*/__pycache__/*' \ + -not -path '*/dist/*' \ + -not -path '*/build/*' \ + -not -path '*/.next/*' + fi +} + +# Dateien verarbeiten +get_file_list | sort | while IFS= read -r file; do + # Überspringe leere Pfade, die Zieldatei selbst oder typische Binär-Endungen + [ -z "$file" ] && continue + [ "$file" = "$OUTPUT_FILE" ] && continue + [ "$(realpath "$file" 2>/dev/null)" = "$OUTPUT_PATH" ] && continue + + case "$file" in + *.png|*.jpg|*.jpeg|*.gif|*.ico|*.pdf|*.zip|*.tar.gz|*.lock|package-lock.json|yarn.lock) + continue + ;; + esac + + # Prüfen, ob die Datei existiert und lesbarer Text ist + if [ -f "$file" ] && (grep -Iq . "$file" 2>/dev/null || [ ! -s "$file" ]); then + echo -e "\n--- DATEI: $file ---" >> "$OUTPUT_PATH" + echo '```' >> "$OUTPUT_PATH" + cat "$file" >> "$OUTPUT_PATH" + echo -e "\n"'```' >> "$OUTPUT_PATH" + fi +done + +echo "Fertig! Projektkontext liegt in: $OUTPUT_PATH" +``` + +--- DATEI: docker-compose.yml --- +``` +version: '3.8' + +services: + redis: + image: redis:alpine + container_name: scraper-redis + restart: always + ports: + - "6379:6379" + + scraper-app: + build: . + container_name: scraper-app + restart: always + ports: + - "3000:3000" + environment: + - REDIS_HOST=redis + - REDIS_PORT=6379 + depends_on: + - redis + +``` + +--- DATEI: package.json --- +``` +{ + "name": "scraper-service", + "version": "1.0.0", + "main": "src/index.js", + "type": "module", + "scripts": { + "start": "node src/index.js" + }, + "dependencies": { + "crawlee": "^3.10.0", + "playwright": "1.45.0", + "express": "^4.19.0", + "bullmq": "^5.8.0", + "axios": "^1.7.0" + }, + "overrides": { + "css-select": "5.1.0" + } +} + +``` + +--- DATEI: src/index.js --- +``` +import { Worker } from 'bullmq'; +import { chromium } from 'playwright'; +import axios from 'axios'; + +const REDIS_CONFIG = { + connection: { + host: process.env.REDIS_HOST || 'redis', + port: parseInt(process.env.REDIS_PORT || '6379', 10) + } +}; + +const WESTERN_COUNTRIES = [ + 'Deutschland', 'Schweiz', 'Österreich', 'Vereinigte Staaten', 'USA', + 'Vereinigtes Königreich', 'Großbritannien', 'Niederlande', 'Frankreich', + 'Spanien', 'Italien', 'Kanada', 'Australien', 'Belgien', 'Schweden', + 'Norwegen', 'Dänemark', 'Finnland', 'Irland', 'Luxemburg', 'Indien' +]; + +const worker = new Worker('scrape-jobs', async (job) => { + const { url, webhookUrl, appWebhookUrl, hotelId } = job.data; + const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http')); + + console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews)' : 'PREIS-SCAN (Schnell)'} für: ${url}`); + + let scrapedPrice = null; + let hasGenius = false; + let roomSizeM2 = null; + let hotelDetails = { name: null, rating: null, reviewCount: null, address: null, image: null }; + let domData = { dNights: 1, dPersons: 2, roomSize: null, checkinDate: null, checkoutDate: null }; + let extractedReviews = []; + let browser = null; + + try { + browser = await chromium.launch({ headless: true }); + const context = await browser.newContext({ + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', + viewport: { width: 1280, height: 900 }, + locale: 'de-DE', + timezoneId: 'Europe/Berlin' + }); + + const page = await context.newPage(); + let targetUrl = url; + if (isReviewScan && !targetUrl.includes('#tab-reviews')) { + targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews'; + } + + await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 35000 }); + await page.waitForTimeout(2500); + + // DATUM AUS URL PARSEN + try { + const parsedUrl = new URL(url); + domData.checkinDate = parsedUrl.searchParams.get('checkin'); + domData.checkoutDate = parsedUrl.searchParams.get('checkout'); + } catch(e) {} + + // 1. JSON-LD STAMMDATEN + hotelDetails = await page.evaluate(() => { + try { + const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]')); + for (const script of scripts) { + try { + const json = JSON.parse(script.innerText || '{}'); + if (json['@type'] === 'Hotel' || json['@type'] === 'LodgingBusiness' || json.name) { + return { + name: json.name || null, + rating: json.aggregateRating?.ratingValue || null, + reviewCount: json.aggregateRating?.reviewCount || null, + address: json.address?.streetAddress || json.address?.addressLocality || null, + image: Array.isArray(json.image) ? json.image[0] : (json.image || null) + }; + } + } catch (e) {} + } + } catch (e) {} + return { name: null, rating: null, reviewCount: null, address: null, image: null }; + }); + + // 2. DOM DATEN + const extraDom = await page.evaluate(() => { + try { + const bodyText = document.body ? (document.body.innerText || '') : ''; + const nMatch = bodyText.match(/(\d+)\s*Nächte?/i); + const pMatch = bodyText.match(/(\d+)\s*(?:Erwachsene|Erwachsener|Gäste|Personen)/i); + const rmMatch = bodyText.match(/(\d+)\s*m²/i); + return { + dNights: nMatch ? parseInt(nMatch[1], 10) : 1, + dPersons: pMatch ? parseInt(pMatch[1], 10) : 2, + roomSize: rmMatch ? parseInt(rmMatch[1], 10) : null + }; + } catch (e) { + return { dNights: 1, dPersons: 2, roomSize: null }; + } + }); + + domData.dNights = extraDom.dNights; + domData.dPersons = extraDom.dPersons; + roomSizeM2 = extraDom.roomSize; + + // PREIS AUSLESEN + scrapedPrice = await page.evaluate(() => { + const selectors = [ + '[data-testid="price-and-discounted-price"]', + '.totalPrice .bui-price-display__value', + '.hprt-price-price', + '.prco-valign-middle-helper' + ]; + for (const sel of selectors) { + const el = document.querySelector(sel); + if (el && el.innerText && /\d/.test(el.innerText)) return el.innerText.trim(); + } + return null; + }); + + // GENIUS BADGE ERKENNUNG (NEU) + hasGenius = await page.evaluate(() => { + const geniusSelectors = [ + '.m-badge__genius', + 'svg.genius-badge', + '.-genius-new_identity-genius_badge', + '.c-deals-container__genius-badge-fix', + '[aria-label*="Genius"]', + '[data-testid="deal-badge"]' + ]; + return geniusSelectors.some(sel => Boolean(document.querySelector(sel))); + }); + + if (hasGenius) { + console.log(`[Job ${job.id}] 💙 Genius-Badge erkannt (-10% Rabatt gilt)!`); + } + + // 3. KOMMENTARE EXTRAHIEREN + if (isReviewScan) { + console.log(`[Job ${job.id}] Starte Review-Extraction mit Pagination...`); + try { + const reviewTab = await page.$('a[href*="#tab-reviews"], #js--hp-gallery-scorecard a, [data-testid="review-score-link"]'); + if (reviewTab) { + await reviewTab.click(); + await page.waitForTimeout(2000); + } + } catch (e) {} + + await page.evaluate(() => window.scrollBy(0, 1000)); + await page.waitForTimeout(2000); + + let pageCount = 0; + const seenKeys = new Set(); + + while (extractedReviews.length < 80 && pageCount < 15) { + const pageReviews = await page.evaluate(({ westernCountries }) => { + const cards = Array.from(document.querySelectorAll('[data-testid="review-card"], [role="group"][aria-label*="Bewertung"], li.review_list_new_item_block, .review_item')); + const results = []; + const currentYear = 2026; + + for (const card of cards) { + const cardText = card ? (card.innerText || '') : ''; + if (!cardText) continue; + + const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase())); + if (!isWestern) continue; + + const yearMatch = cardText.match(/\b(20\d{2})\b/); + if (yearMatch) { + const reviewYear = parseInt(yearMatch[1], 10); + if (currentYear - reviewYear > 2) continue; + } + + const titleEl = card.querySelector('[data-testid="review-title"], h4'); + const posEl = card.querySelector('[data-testid="review-positive-text"], .review_pos'); + const negEl = card.querySelector('[data-testid="review-negative-text"], .review_neg'); + const dateEl = card.querySelector('[data-testid="review-date"]'); + + const title = titleEl ? titleEl.innerText.trim() : ''; + const positive = posEl ? posEl.innerText.trim() : ''; + const negative = negEl ? negEl.innerText.trim() : ''; + const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell'); + + let fallbackText = ''; + if (!positive && !negative) fallbackText = cardText.replace(title, '').substring(0, 300).trim(); + + if (title || positive || negative || fallbackText) { + results.push({ title, positive: positive || fallbackText, negative, date: dateText }); + } + } + return results; + }, { westernCountries: WESTERN_COUNTRIES }).catch(() => []); + + for (const rev of pageReviews) { + const key = `${rev.title}-${rev.positive}-${rev.date}`; + if (!seenKeys.has(key) && extractedReviews.length < 80) { + seenKeys.add(key); + extractedReviews.push(rev); + } + } + + if (extractedReviews.length >= 80) break; + + try { + const nextBtn = await page.$('#reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"]'); + if (nextBtn) { + const isDisabled = await nextBtn.getAttribute('disabled'); + if (isDisabled !== null) break; + await nextBtn.click(); + await page.waitForTimeout(2000); + pageCount++; + } else break; + } catch (e) { break; } + } + console.log(`[Job ${job.id}] ${extractedReviews.length} Kommentare extrahiert.`); + } + + } catch (error) { + console.error(`[Job ${job.id}] Fehler beim Scraping:`, error.message); + } finally { + if (browser) await browser.close(); + + // 1. PREISHISTORIE + DATUM SPEICHERN + if (appWebhookUrl) { + try { + await axios.post(appWebhookUrl, { + price: scrapedPrice, + hasGenius: hasGenius, + roomSizeM2, + hotelName: hotelDetails.name, + rating: hotelDetails.rating, + reviewCount: hotelDetails.reviewCount, + address: hotelDetails.address, + image: hotelDetails.image, + checkinDate: domData.checkinDate, + checkoutDate: domData.checkoutDate, + nights: domData.dNights || 1, + persons: domData.dPersons || 2, + }); + console.log(`[Job ${job.id}] ✅ Preishistorie in Postgres gespeichert!`); + } catch (e) { + console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie:`, e.message); + } + } + + // 2. REVIEWS AN N8N SENDEN + if (isReviewScan && extractedReviews.length > 0) { + try { + await axios.post(webhookUrl, { + jobId: job.id, + hotelId, + hotelName: hotelDetails.name, + reviews: extractedReviews, + timestamp: new Date().toISOString() + }); + console.log(`[Job ${job.id}] 🚀 ${extractedReviews.length} Reviews an n8n gesendet!`); + } catch (e) {} + } + } + +}, { ...REDIS_CONFIG, concurrency: 2 }); + +console.log('🚀 Scraper Worker bereit...'); + +``` diff --git a/src/index.js b/src/index.js index af516c5..b1efb20 100644 --- a/src/index.js +++ b/src/index.js @@ -18,13 +18,15 @@ const WESTERN_COUNTRIES = [ const worker = new Worker('scrape-jobs', async (job) => { const { url, webhookUrl, appWebhookUrl, hotelId } = job.data; - console.log(`[Job ${job.id}] Starte Scraping für: ${url}`); + const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http')); + + console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews)' : 'PREIS-SCAN (Schnell)'} für: ${url}`); - // Variablen auf Worker-Ebene deklarieren (Verhindert ReferenceError in finally) let scrapedPrice = null; + let hasGenius = false; let roomSizeM2 = null; let hotelDetails = { name: null, rating: null, reviewCount: null, address: null, image: null }; - let domData = { dNights: 1, dPersons: 2, roomSize: null }; + let domData = { dNights: 1, dPersons: 2, roomSize: null, checkinDate: null, checkoutDate: null }; let extractedReviews = []; let browser = null; @@ -38,212 +40,206 @@ const worker = new Worker('scrape-jobs', async (job) => { }); const page = await context.newPage(); - let targetUrl = url; - if (!targetUrl.includes('#tab-reviews')) { + if (isReviewScan && !targetUrl.includes('#tab-reviews')) { targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews'; } - if (!targetUrl.includes('sort_by=')) { - targetUrl += (targetUrl.includes('?') ? '&' : '?') + 'sort_by=f_recent_desc'; - } await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 35000 }); await page.waitForTimeout(2500); - // Klick auf Bewertungs-Reiter erzwingen + // DATUM AUS URL PARSEN try { - const reviewTab = await page.$('#js--hp-gallery-scorecard a, a[href*="#tab-reviews"], [data-testid="review-score-link"]'); - if (reviewTab) { - await reviewTab.click(); - await page.waitForTimeout(2000); - } - } catch (e) {} + const parsedUrl = new URL(url); + domData.checkinDate = parsedUrl.searchParams.get('checkin'); + domData.checkoutDate = parsedUrl.searchParams.get('checkout'); + } catch(e) {} - // Scrollen - await page.evaluate(() => { - const el = document.querySelector('#reviewListSorters') || document.querySelector('#tab-reviews'); - if (el) el.scrollIntoView(); - else window.scrollBy(0, 1000); - }); - await page.waitForTimeout(2500); - - // 1. JSON-LD Stammdaten + // 1. JSON-LD STAMMDATEN hotelDetails = await page.evaluate(() => { - const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]')); - for (const script of scripts) { - try { - const json = JSON.parse(script.innerText); - if (json['@type'] === 'Hotel' || json['@type'] === 'LodgingBusiness' || json.name) { - return { - name: json.name || null, - rating: json.aggregateRating?.ratingValue || null, - reviewCount: json.aggregateRating?.reviewCount || null, - address: json.address?.streetAddress || json.address?.addressLocality || null, - image: json.image || null - }; - } - } catch (e) {} - } + try { + const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]')); + for (const script of scripts) { + try { + const json = JSON.parse(script.innerText || '{}'); + if (json['@type'] === 'Hotel' || json['@type'] === 'LodgingBusiness' || json.name) { + return { + name: json.name || null, + rating: json.aggregateRating?.ratingValue || null, + reviewCount: json.aggregateRating?.reviewCount || null, + address: json.address?.streetAddress || json.address?.addressLocality || null, + image: Array.isArray(json.image) ? json.image[0] : (json.image || null) + }; + } + } catch (e) {} + } + } catch (e) {} return { name: null, rating: null, reviewCount: null, address: null, image: null }; }); - // 2. Preis & DOM-Daten - domData = await page.evaluate(() => { - let dNights = null; - let dPersons = null; - const targetSelectors = ['.bui-price-display__label', '#searchbox-horizontal-occupancy-label', '[data-testid="price-for-x-nights"]']; - let combinedText = ''; - targetSelectors.forEach(s => document.querySelectorAll(s).forEach(el => combinedText += ' ' + el.innerText)); - if (!combinedText.trim()) combinedText = document.body.innerText; - - const nMatch = combinedText.match(/(\d+)\s*Nächte?/i); - if (nMatch) dNights = parseInt(nMatch[1], 10); - const pMatch = combinedText.match(/(\d+)\s*(?:Erwachsene|Erwachsener|Gäste|Personen)/i); - if (pMatch) dPersons = parseInt(pMatch[1], 10); - const rmMatch = document.body.innerText.match(/(\d+)\s*m²/i); - - return { dNights, dPersons, roomSize: rmMatch ? parseInt(rmMatch[1], 10) : null }; + // 2. DOM DATEN + const extraDom = await page.evaluate(() => { + try { + const bodyText = document.body ? (document.body.innerText || '') : ''; + const nMatch = bodyText.match(/(\d+)\s*Nächte?/i); + const pMatch = bodyText.match(/(\d+)\s*(?:Erwachsene|Erwachsener|Gäste|Personen)/i); + const rmMatch = bodyText.match(/(\d+)\s*m²/i); + return { + dNights: nMatch ? parseInt(nMatch[1], 10) : 1, + dPersons: pMatch ? parseInt(pMatch[1], 10) : 2, + roomSize: rmMatch ? parseInt(rmMatch[1], 10) : null + }; + } catch (e) { + return { dNights: 1, dPersons: 2, roomSize: null }; + } }); - roomSizeM2 = domData.roomSize; + domData.dNights = extraDom.dNights; + domData.dPersons = extraDom.dPersons; + roomSizeM2 = extraDom.roomSize; - const priceSelectors = ['[data-testid="price-and-discounted-price"]', '.totalPrice .bui-price-display__value', '.hprt-price-price']; - for (const sel of priceSelectors) { - const el = await page.$(sel); - if (el) { - const text = await el.innerText(); - if (text && /\d/.test(text)) { scrapedPrice = text.trim(); break; } + // PREIS AUSLESEN + scrapedPrice = await page.evaluate(() => { + const selectors = [ + '[data-testid="price-and-discounted-price"]', + '.totalPrice .bui-price-display__value', + '.hprt-price-price', + '.prco-valign-middle-helper' + ]; + for (const sel of selectors) { + const el = document.querySelector(sel); + if (el && el.innerText && /\d/.test(el.innerText)) return el.innerText.trim(); } + return null; + }); + + // GENIUS BADGE ERKENNUNG (NEU) + hasGenius = await page.evaluate(() => { + const geniusSelectors = [ + '.m-badge__genius', + 'svg.genius-badge', + '.-genius-new_identity-genius_badge', + '.c-deals-container__genius-badge-fix', + '[aria-label*="Genius"]', + '[data-testid="deal-badge"]' + ]; + return geniusSelectors.some(sel => Boolean(document.querySelector(sel))); + }); + + if (hasGenius) { + console.log(`[Job ${job.id}] 💙 Genius-Badge erkannt (-10% Rabatt gilt)!`); } - // 3. KOMMENTARE MIT PAGINATION BIS ZU 80 STÜCK EXTRAHIEREN - let pageCount = 0; - const seenKeys = new Set(); - - while (extractedReviews.length < 80 && pageCount < 15) { - const pageReviews = await page.evaluate(({ westernCountries }) => { - const dateElements = Array.from(document.querySelectorAll('[data-testid="review-date"]')); - const results = []; - const currentYear = 2026; - - for (const dateEl of dateElements) { - // Navigiere hoch zum Container, der SOWOHL Person/Land ALS AUCH Bewertungstext enthält - let card = dateEl.parentElement; - while (card && card !== document.body) { - if (card.querySelector('[data-testid="review-avatar"]') || card.querySelector('[aria-label="Bewertende Person"]')) { - break; - } - card = card.parentElement; - } - if (!card || card === document.body) { - card = dateEl.closest('li') || dateEl.parentElement.parentElement.parentElement; - } - - const cardText = card ? card.innerText : ''; - if (!cardText) continue; - - // Herkunftsland prüfen - const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase())); - if (!isWestern) continue; - - // Datum prüfen (<= 2 Jahre) - const dateText = dateEl.innerText || ''; - const yearMatch = cardText.match(/\b(20\d{2})\b/); - if (yearMatch) { - const reviewYear = parseInt(yearMatch[1], 10); - if (currentYear - reviewYear > 2) continue; - } - - const titleEl = card.querySelector('[data-testid="review-title"]'); - const posEl = card.querySelector('[data-testid="review-positive-text"], .review_pos'); - const negEl = card.querySelector('[data-testid="review-negative-text"], .review_neg'); - - const title = titleEl ? titleEl.innerText.trim() : ''; - const positive = posEl ? posEl.innerText.trim() : ''; - const negative = negEl ? negEl.innerText.trim() : ''; - - let generalComment = ''; - if (!positive && !negative) { - const commentEl = card.querySelector('.b99b6ef58f, .a17eaf5728'); - if (commentEl && !commentEl.innerText.includes('keinen Kommentar')) { - generalComment = commentEl.innerText.trim(); - } - } - - if (title || positive || negative || generalComment) { - results.push({ - title, - positive: positive || generalComment, - negative, - date: dateText - }); - } - } - - return results; - }, { westernCountries: WESTERN_COUNTRIES }); - - // Duplikate vermeiden - for (const rev of pageReviews) { - const key = `${rev.title}-${rev.positive}-${rev.date}`; - if (!seenKeys.has(key) && extractedReviews.length < 80) { - seenKeys.add(key); - extractedReviews.push(rev); - } - } - - if (extractedReviews.length >= 80) break; - - // Klick auf "Nächste Seite" Pfeil + // 3. KOMMENTARE EXTRAHIEREN + if (isReviewScan) { + console.log(`[Job ${job.id}] Starte Review-Extraction mit Pagination...`); try { - const nextBtn = await page.$( - '#reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"], [data-testid="pagination-next-page"]' - ); - - if (nextBtn) { - const isDisabled = await nextBtn.getAttribute('disabled'); - if (isDisabled !== null) break; - - await nextBtn.click(); - await page.waitForTimeout(2500); - pageCount++; - } else { - break; + const reviewTab = await page.$('a[href*="#tab-reviews"], #js--hp-gallery-scorecard a, [data-testid="review-score-link"]'); + if (reviewTab) { + await reviewTab.click(); + await page.waitForTimeout(2000); } - } catch (e) { - break; - } - } + } catch (e) {} - console.log(`[Job ${job.id}] ${extractedReviews.length} Kommentare über ${pageCount + 1} Seiten extrahiert.`); + await page.evaluate(() => window.scrollBy(0, 1000)); + await page.waitForTimeout(2000); + + let pageCount = 0; + const seenKeys = new Set(); + + while (extractedReviews.length < 80 && pageCount < 15) { + const pageReviews = await page.evaluate(({ westernCountries }) => { + const cards = Array.from(document.querySelectorAll('[data-testid="review-card"], [role="group"][aria-label*="Bewertung"], li.review_list_new_item_block, .review_item')); + const results = []; + const currentYear = 2026; + + for (const card of cards) { + const cardText = card ? (card.innerText || '') : ''; + if (!cardText) continue; + + const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase())); + if (!isWestern) continue; + + const yearMatch = cardText.match(/\b(20\d{2})\b/); + if (yearMatch) { + const reviewYear = parseInt(yearMatch[1], 10); + if (currentYear - reviewYear > 2) continue; + } + + const titleEl = card.querySelector('[data-testid="review-title"], h4'); + const posEl = card.querySelector('[data-testid="review-positive-text"], .review_pos'); + const negEl = card.querySelector('[data-testid="review-negative-text"], .review_neg'); + const dateEl = card.querySelector('[data-testid="review-date"]'); + + const title = titleEl ? titleEl.innerText.trim() : ''; + const positive = posEl ? posEl.innerText.trim() : ''; + const negative = negEl ? negEl.innerText.trim() : ''; + const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell'); + + let fallbackText = ''; + if (!positive && !negative) fallbackText = cardText.replace(title, '').substring(0, 300).trim(); + + if (title || positive || negative || fallbackText) { + results.push({ title, positive: positive || fallbackText, negative, date: dateText }); + } + } + return results; + }, { westernCountries: WESTERN_COUNTRIES }).catch(() => []); + + for (const rev of pageReviews) { + const key = `${rev.title}-${rev.positive}-${rev.date}`; + if (!seenKeys.has(key) && extractedReviews.length < 80) { + seenKeys.add(key); + extractedReviews.push(rev); + } + } + + if (extractedReviews.length >= 80) break; + + try { + const nextBtn = await page.$('#reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"]'); + if (nextBtn) { + const isDisabled = await nextBtn.getAttribute('disabled'); + if (isDisabled !== null) break; + await nextBtn.click(); + await page.waitForTimeout(2000); + pageCount++; + } else break; + } catch (e) { break; } + } + console.log(`[Job ${job.id}] ${extractedReviews.length} Kommentare extrahiert.`); + } } catch (error) { console.error(`[Job ${job.id}] Fehler beim Scraping:`, error.message); } finally { if (browser) await browser.close(); - // 1. PREISHISTORIE IN POSTGRES SPEICHERN + // 1. PREISHISTORIE + DATUM SPEICHERN if (appWebhookUrl) { try { await axios.post(appWebhookUrl, { price: scrapedPrice, + hasGenius: hasGenius, roomSizeM2, hotelName: hotelDetails.name, rating: hotelDetails.rating, reviewCount: hotelDetails.reviewCount, address: hotelDetails.address, image: hotelDetails.image, + checkinDate: domData.checkinDate, + checkoutDate: domData.checkoutDate, nights: domData.dNights || 1, persons: domData.dPersons || 2, }); - console.log(`[Job ${job.id}] ✅ Preishistorie direkt in Postgres gespeichert!`); + console.log(`[Job ${job.id}] ✅ Preishistorie in Postgres gespeichert!`); } catch (e) { - console.error(`[Job ${job.id}] ❌ Fehler beim Speichern der Preishistorie:`, e.message); + console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie:`, e.message); } } // 2. REVIEWS AN N8N SENDEN - if (webhookUrl) { + if (isReviewScan && extractedReviews.length > 0) { try { await axios.post(webhookUrl, { jobId: job.id, @@ -253,9 +249,7 @@ const worker = new Worker('scrape-jobs', async (job) => { timestamp: new Date().toISOString() }); console.log(`[Job ${job.id}] 🚀 ${extractedReviews.length} Reviews an n8n gesendet!`); - } catch (e) { - console.error(`[Job ${job.id}] ❌ Fehler beim Senden an n8n:`, e.message); - } + } catch (e) {} } }