diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6c19dd9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM mcr.microsoft.com/playwright:v1.45.0-jammy + +WORKDIR /app + +# Package-Dateien kopieren und installieren +COPY package*.json ./ +RUN npm install + +# Sourcecode kopieren +COPY . . + +EXPOSE 3000 + +CMD ["node", "src/index.js"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f1d5454 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3.8' + +services: + redis: + image: redis:alpine + container_name: scraper-redis + restart: always + ports: + - "6379:6379" + + scraper-app: + build: . + container_name: scraper-app + restart: always + ports: + - "3000:3000" + environment: + - REDIS_HOST=redis + - REDIS_PORT=6379 + depends_on: + - redis diff --git a/package.json b/package.json new file mode 100644 index 0000000..cb083e2 --- /dev/null +++ b/package.json @@ -0,0 +1,19 @@ +{ + "name": "scraper-service", + "version": "1.0.0", + "main": "src/index.js", + "type": "module", + "scripts": { + "start": "node src/index.js" + }, + "dependencies": { + "crawlee": "^3.10.0", + "playwright": "1.45.0", + "express": "^4.19.0", + "bullmq": "^5.8.0", + "axios": "^1.7.0" + }, + "overrides": { + "css-select": "5.1.0" + } +} diff --git a/src/index.js b/src/index.js new file mode 100644 index 0000000..9fd50c8 --- /dev/null +++ b/src/index.js @@ -0,0 +1,145 @@ +const worker = new Worker('scrape-jobs', async (job) => { + const { url, webhookUrl } = job.data; + console.log(`[Job ${job.id}] Starte Scraping für: ${url}`); + + let scrapedPrice = null; + let roomSizeM2 = null; + let hotelDetails = { name: null, rating: null, reviewCount: null, address: null, image: null }; + let browser = null; + + // 1. Fallback: Nächte aus URL-Checkin/Checkout berechnen + const urlObj = new URL(url); + const checkin = urlObj.searchParams.get('checkin'); + const checkout = urlObj.searchParams.get('checkout'); + const adultsParam = parseInt(urlObj.searchParams.get('group_adults') || '0', 10); + + let nightsFromUrl = null; + if (checkin && checkout) { + const diffTime = Math.abs(new Date(checkout) - new Date(checkin)); + nightsFromUrl = Math.ceil(diffTime / (1000 * 60 * 60 * 24)); + } + + try { + browser = await chromium.launch({ headless: true }); + const context = await browser.newContext({ + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', + viewport: { width: 1280, height: 800 }, + locale: 'de-DE', + timezoneId: 'Europe/Berlin' + }); + + const page = await context.newPage(); + await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 35000 }); + await page.waitForTimeout(3000); + + // 2. Hotel Details aus JSON-LD + hotelDetails = await page.evaluate(() => { + const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]')); + for (const script of scripts) { + try { + const json = JSON.parse(script.innerText); + if (json['@type'] === 'Hotel' || json['@type'] === 'LodgingBusiness' || json.name) { + return { + name: json.name || null, + rating: json.aggregateRating?.ratingValue || null, + reviewCount: json.aggregateRating?.reviewCount || null, + address: json.address?.streetAddress || json.address?.addressLocality || null, + image: json.image || null + }; + } + } catch (e) {} + } + return { name: null, rating: null, reviewCount: null, address: null, image: null }; + }); + + // 3. Nächte, Personen & Zimmergröße direkt aus den Booking-DOM-Selektoren lesen + const domData = await page.evaluate(() => { + let dNights = null; + let dPersons = null; + + // Gezielte Suche in den von dir genannten Selektoren + const targetSelectors = [ + '.bui-price-display__label', + '#searchbox-horizontal-occupancy-label', + '[data-testid="price-for-x-nights"]', + '#group_recommendation .bui-price-display__label', + '[data-testid="occupancy-config"]' + ]; + + let combinedText = ''; + for (const sel of targetSelectors) { + const els = document.querySelectorAll(sel); + els.forEach(el => { + if (el && el.innerText) combinedText += ' ' + el.innerText; + }); + } + + // Falls Selektoren leer sind, gesamten Text als Fallback nutzen + if (!combinedText.trim()) { + combinedText = document.body.innerText; + } + + // Nächte matchen (z. B. "7 Nächte", "1 Nacht", "für 2 Nächte") + const nightMatch = combinedText.match(/(\d+)\s*Nächte?/i); + if (nightMatch) dNights = parseInt(nightMatch[1], 10); + + // Personen matchen (z. B. "2 Erwachsene", "1 Erwachsener", "2 Gäste") + const personMatch = combinedText.match(/(\d+)\s*(?:Erwachsene|Erwachsener|Gäste|Personen)/i); + if (personMatch) dPersons = parseInt(personMatch[1], 10); + + // Zimmergröße (m²) + const roomMatch = document.body.innerText.match(/(\d+)\s*m²/i); + const roomSize = roomMatch ? parseInt(roomMatch[1], 10) : null; + + return { dNights, dPersons, roomSize }; + }); + + roomSizeM2 = domData.roomSize; + + // Finale Werte bestimmen (Priorität: URL -> DOM -> Standard-Fallback) + const finalNights = nightsFromUrl || domData.dNights || 1; + const finalPersons = adultsParam || domData.dPersons || 2; + + // 4. Preis extrahieren + const priceSelectors = [ + '[data-testid="price-and-discounted-price"]', + '.totalPrice .bui-price-display__value', + '.hprt-price-price' + ]; + + for (const selector of priceSelectors) { + const el = await page.$(selector); + if (el) { + const text = await el.innerText(); + if (text && /\d/.test(text)) { + scrapedPrice = text.trim(); + break; + } + } + } + + // 5. Webhook mit exakten Daten absenden + if (webhookUrl && webhookUrl.startsWith('http')) { + await axios.post(webhookUrl, { + jobId: job.id, + url, + hotelName: hotelDetails.name, + price: scrapedPrice, + roomSizeM2, + rating: hotelDetails.rating, + reviewCount: hotelDetails.reviewCount, + address: hotelDetails.address, + image: hotelDetails.image, + nights: finalNights, + persons: finalPersons, + timestamp: new Date().toISOString() + }); + } + + } catch (error) { + console.error(`[Job ${job.id}] Fehler:`, error.message); + } finally { + if (browser) await browser.close(); + } + +}, { ...REDIS_CONFIG, concurrency: 2 });