======================================== PROJEKTSTRUKTUR ======================================== (Ermittelt via Git unter Berücksichtigung von .gitignore) copyChat.sh llm_context.txt Dockerfile README.md docker-compose.yml package.json src/index.js ======================================== DATEIINHALTE ======================================== --- DATEI: Dockerfile --- ``` FROM mcr.microsoft.com/playwright:v1.45.0-jammy WORKDIR /app # Package-Dateien kopieren und installieren COPY package*.json ./ RUN npm install # Sourcecode kopieren COPY . . EXPOSE 3000 CMD ["node", "src/index.js"] ``` --- DATEI: README.md --- ``` ``` --- DATEI: copyChat.sh --- ``` #!/usr/bin/env bash OUTPUT_FILE="${1:-llm_context.txt}" ROOT_DIR="${2:-.}" # Absolute Pfade auflösen OUTPUT_PATH="$(cd "$(dirname "$OUTPUT_FILE")" 2>/dev/null && pwd)/$(basename "$OUTPUT_FILE")" cd "$ROOT_DIR" || exit 1 # Ausgabedatei initialisieren / leeren > "$OUTPUT_PATH" echo "========================================" >> "$OUTPUT_PATH" echo "PROJEKTSTRUKTUR" >> "$OUTPUT_PATH" echo "========================================" >> "$OUTPUT_PATH" # Prüfen, ob Git verfügbar und das Verzeichnis ein Git-Repo ist IS_GIT_REPO=false if command -v git &> /dev/null && git rev-parse --is-inside-work-tree &> /dev/null; then IS_GIT_REPO=true fi if [ "$IS_GIT_REPO" = true ]; then echo "(Ermittelt via Git unter Berücksichtigung von .gitignore)" >> "$OUTPUT_PATH" # Zeigt alle getrackten + ungetrackten Dateien an, ignoriert .gitignore-Treffer git ls-files -co --exclude-standard >> "$OUTPUT_PATH" else if command -v tree &> /dev/null; then tree -I 'node_modules|.git|venv|.venv|__pycache__|dist|build|.next|target' >> "$OUTPUT_PATH" else find . -maxdepth 3 -not -path '*/.*' -not -path '*/node_modules*' -not -path '*/venv*' >> "$OUTPUT_PATH" fi fi echo -e "\n========================================" >> "$OUTPUT_PATH" echo "DATEIINHALTE" >> "$OUTPUT_PATH" echo "========================================" >> "$OUTPUT_PATH" # Dateiliste ermitteln get_file_list() { if [ "$IS_GIT_REPO" = true ]; then # -c = cached/tracked, -o = others/untracked, --exclude-standard = .gitignore beachten git ls-files -co --exclude-standard else find . -type f \ -not -path '*/.git/*' \ -not -path '*/node_modules/*' \ -not -path '*/venv/*' \ -not -path '*/.venv/*' \ -not -path '*/__pycache__/*' \ -not -path '*/dist/*' \ -not -path '*/build/*' \ -not -path '*/.next/*' fi } # Dateien verarbeiten get_file_list | sort | while IFS= read -r file; do # Überspringe leere Pfade, die Zieldatei selbst oder typische Binär-Endungen [ -z "$file" ] && continue [ "$file" = "$OUTPUT_FILE" ] && continue [ "$(realpath "$file" 2>/dev/null)" = "$OUTPUT_PATH" ] && continue case "$file" in *.png|*.jpg|*.jpeg|*.gif|*.ico|*.pdf|*.zip|*.tar.gz|*.lock|package-lock.json|yarn.lock) continue ;; esac # Prüfen, ob die Datei existiert und lesbarer Text ist if [ -f "$file" ] && (grep -Iq . "$file" 2>/dev/null || [ ! -s "$file" ]); then echo -e "\n--- DATEI: $file ---" >> "$OUTPUT_PATH" echo '```' >> "$OUTPUT_PATH" cat "$file" >> "$OUTPUT_PATH" echo -e "\n"'```' >> "$OUTPUT_PATH" fi done echo "Fertig! Projektkontext liegt in: $OUTPUT_PATH" ``` --- DATEI: docker-compose.yml --- ``` version: '3.8' services: redis: image: redis:alpine container_name: scraper-redis restart: always ports: - "6379:6379" scraper-app: build: . container_name: scraper-app restart: always ports: - "3000:3000" environment: - REDIS_HOST=redis - REDIS_PORT=6379 depends_on: - redis ``` --- DATEI: package.json --- ``` { "name": "scraper-service", "version": "1.0.0", "main": "src/index.js", "type": "module", "scripts": { "start": "node src/index.js" }, "dependencies": { "crawlee": "^3.10.0", "playwright": "1.45.0", "express": "^4.19.0", "bullmq": "^5.8.0", "axios": "^1.7.0" }, "overrides": { "css-select": "5.1.0" } } ``` --- DATEI: src/index.js --- ``` import { Worker } from 'bullmq'; import { chromium } from 'playwright'; import axios from 'axios'; const REDIS_CONFIG = { connection: { host: process.env.REDIS_HOST || 'redis', port: parseInt(process.env.REDIS_PORT || '6379', 10) } }; const WESTERN_COUNTRIES = [ 'Deutschland', 'Schweiz', 'Österreich', 'Vereinigte Staaten', 'USA', 'Vereinigtes Königreich', 'Großbritannien', 'Niederlande', 'Frankreich', 'Spanien', 'Italien', 'Kanada', 'Australien', 'Belgien', 'Schweden', 'Norwegen', 'Dänemark', 'Finnland', 'Irland', 'Luxemburg', 'Indien' ]; const worker = new Worker('scrape-jobs', async (job) => { const { url, webhookUrl, appWebhookUrl, hotelId } = job.data; const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http')); console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews)' : 'PREIS-SCAN (Schnell)'} für: ${url}`); let scrapedPrice = null; let hasGenius = false; let roomSizeM2 = null; let hotelDetails = { name: null, rating: null, reviewCount: null, address: null, image: null }; let domData = { dNights: 1, dPersons: 2, roomSize: null, checkinDate: null, checkoutDate: null }; let extractedReviews = []; let browser = null; try { browser = await chromium.launch({ headless: true }); const context = await browser.newContext({ userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', viewport: { width: 1280, height: 900 }, locale: 'de-DE', timezoneId: 'Europe/Berlin' }); const page = await context.newPage(); let targetUrl = url; if (isReviewScan && !targetUrl.includes('#tab-reviews')) { targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews'; } await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 35000 }); await page.waitForTimeout(2500); // DATUM AUS URL PARSEN try { const parsedUrl = new URL(url); domData.checkinDate = parsedUrl.searchParams.get('checkin'); domData.checkoutDate = parsedUrl.searchParams.get('checkout'); } catch(e) {} // 1. JSON-LD STAMMDATEN hotelDetails = await page.evaluate(() => { try { const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]')); for (const script of scripts) { try { const json = JSON.parse(script.innerText || '{}'); if (json['@type'] === 'Hotel' || json['@type'] === 'LodgingBusiness' || json.name) { return { name: json.name || null, rating: json.aggregateRating?.ratingValue || null, reviewCount: json.aggregateRating?.reviewCount || null, address: json.address?.streetAddress || json.address?.addressLocality || null, image: Array.isArray(json.image) ? json.image[0] : (json.image || null) }; } } catch (e) {} } } catch (e) {} return { name: null, rating: null, reviewCount: null, address: null, image: null }; }); // 2. DOM DATEN const extraDom = await page.evaluate(() => { try { const bodyText = document.body ? (document.body.innerText || '') : ''; const nMatch = bodyText.match(/(\d+)\s*Nächte?/i); const pMatch = bodyText.match(/(\d+)\s*(?:Erwachsene|Erwachsener|Gäste|Personen)/i); const rmMatch = bodyText.match(/(\d+)\s*m²/i); return { dNights: nMatch ? parseInt(nMatch[1], 10) : 1, dPersons: pMatch ? parseInt(pMatch[1], 10) : 2, roomSize: rmMatch ? parseInt(rmMatch[1], 10) : null }; } catch (e) { return { dNights: 1, dPersons: 2, roomSize: null }; } }); domData.dNights = extraDom.dNights; domData.dPersons = extraDom.dPersons; roomSizeM2 = extraDom.roomSize; // PREIS AUSLESEN scrapedPrice = await page.evaluate(() => { const selectors = [ '[data-testid="price-and-discounted-price"]', '.totalPrice .bui-price-display__value', '.hprt-price-price', '.prco-valign-middle-helper' ]; for (const sel of selectors) { const el = document.querySelector(sel); if (el && el.innerText && /\d/.test(el.innerText)) return el.innerText.trim(); } return null; }); // GENIUS BADGE ERKENNUNG (NEU) hasGenius = await page.evaluate(() => { const geniusSelectors = [ '.m-badge__genius', 'svg.genius-badge', '.-genius-new_identity-genius_badge', '.c-deals-container__genius-badge-fix', '[aria-label*="Genius"]', '[data-testid="deal-badge"]' ]; return geniusSelectors.some(sel => Boolean(document.querySelector(sel))); }); if (hasGenius) { console.log(`[Job ${job.id}] 💙 Genius-Badge erkannt (-10% Rabatt gilt)!`); } // 3. KOMMENTARE EXTRAHIEREN if (isReviewScan) { console.log(`[Job ${job.id}] Starte Review-Extraction mit Pagination...`); try { const reviewTab = await page.$('a[href*="#tab-reviews"], #js--hp-gallery-scorecard a, [data-testid="review-score-link"]'); if (reviewTab) { await reviewTab.click(); await page.waitForTimeout(2000); } } catch (e) {} await page.evaluate(() => window.scrollBy(0, 1000)); await page.waitForTimeout(2000); let pageCount = 0; const seenKeys = new Set(); while (extractedReviews.length < 80 && pageCount < 15) { const pageReviews = await page.evaluate(({ westernCountries }) => { const cards = Array.from(document.querySelectorAll('[data-testid="review-card"], [role="group"][aria-label*="Bewertung"], li.review_list_new_item_block, .review_item')); const results = []; const currentYear = 2026; for (const card of cards) { const cardText = card ? (card.innerText || '') : ''; if (!cardText) continue; const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase())); if (!isWestern) continue; const yearMatch = cardText.match(/\b(20\d{2})\b/); if (yearMatch) { const reviewYear = parseInt(yearMatch[1], 10); if (currentYear - reviewYear > 2) continue; } const titleEl = card.querySelector('[data-testid="review-title"], h4'); const posEl = card.querySelector('[data-testid="review-positive-text"], .review_pos'); const negEl = card.querySelector('[data-testid="review-negative-text"], .review_neg'); const dateEl = card.querySelector('[data-testid="review-date"]'); const title = titleEl ? titleEl.innerText.trim() : ''; const positive = posEl ? posEl.innerText.trim() : ''; const negative = negEl ? negEl.innerText.trim() : ''; const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell'); let fallbackText = ''; if (!positive && !negative) fallbackText = cardText.replace(title, '').substring(0, 300).trim(); if (title || positive || negative || fallbackText) { results.push({ title, positive: positive || fallbackText, negative, date: dateText }); } } return results; }, { westernCountries: WESTERN_COUNTRIES }).catch(() => []); for (const rev of pageReviews) { const key = `${rev.title}-${rev.positive}-${rev.date}`; if (!seenKeys.has(key) && extractedReviews.length < 80) { seenKeys.add(key); extractedReviews.push(rev); } } if (extractedReviews.length >= 80) break; try { const nextBtn = await page.$('#reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"]'); if (nextBtn) { const isDisabled = await nextBtn.getAttribute('disabled'); if (isDisabled !== null) break; await nextBtn.click(); await page.waitForTimeout(2000); pageCount++; } else break; } catch (e) { break; } } console.log(`[Job ${job.id}] ${extractedReviews.length} Kommentare extrahiert.`); } } catch (error) { console.error(`[Job ${job.id}] Fehler beim Scraping:`, error.message); } finally { if (browser) await browser.close(); // 1. PREISHISTORIE + DATUM SPEICHERN if (appWebhookUrl) { try { await axios.post(appWebhookUrl, { price: scrapedPrice, hasGenius: hasGenius, roomSizeM2, hotelName: hotelDetails.name, rating: hotelDetails.rating, reviewCount: hotelDetails.reviewCount, address: hotelDetails.address, image: hotelDetails.image, checkinDate: domData.checkinDate, checkoutDate: domData.checkoutDate, nights: domData.dNights || 1, persons: domData.dPersons || 2, }); console.log(`[Job ${job.id}] ✅ Preishistorie in Postgres gespeichert!`); } catch (e) { console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie:`, e.message); } } // 2. REVIEWS AN N8N SENDEN if (isReviewScan && extractedReviews.length > 0) { try { await axios.post(webhookUrl, { jobId: job.id, hotelId, hotelName: hotelDetails.name, reviews: extractedReviews, timestamp: new Date().toISOString() }); console.log(`[Job ${job.id}] 🚀 ${extractedReviews.length} Reviews an n8n gesendet!`); } catch (e) {} } } }, { ...REDIS_CONFIG, concurrency: 2 }); console.log('🚀 Scraper Worker bereit...'); ```