Booking scraper service

Dockerfile

@@ -0,0 +1,14 @@
+FROM mcr.microsoft.com/playwright:v1.45.0-jammy
+
+WORKDIR /app
+
+# Package-Dateien kopieren und installieren
+COPY package*.json ./
+RUN npm install
+
+# Sourcecode kopieren
+COPY . .
+
+EXPOSE 3000
+
+CMD ["node", "src/index.js"]

docker-compose.yml

@@ -0,0 +1,21 @@
+version: '3.8'
+
+services:
+  redis:
+    image: redis:alpine
+    container_name: scraper-redis
+    restart: always
+    ports:
+      - "6379:6379"
+
+  scraper-app:
+    build: .
+    container_name: scraper-app
+    restart: always
+    ports:
+      - "3000:3000"
+    environment:
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+    depends_on:
+      - redis

package.json

@@ -0,0 +1,19 @@
+{
+  "name": "scraper-service",
+  "version": "1.0.0",
+  "main": "src/index.js",
+  "type": "module",
+  "scripts": {
+    "start": "node src/index.js"
+  },
+  "dependencies": {
+    "crawlee": "^3.10.0",
+    "playwright": "1.45.0",
+    "express": "^4.19.0",
+    "bullmq": "^5.8.0",
+    "axios": "^1.7.0"
+  },
+  "overrides": {
+    "css-select": "5.1.0"
+  }
+}

src/index.js

@@ -0,0 +1,145 @@
+const worker = new Worker('scrape-jobs', async (job) => {
+  const { url, webhookUrl } = job.data;
+  console.log(`[Job ${job.id}] Starte Scraping für: ${url}`);
+
+  let scrapedPrice = null;
+  let roomSizeM2 = null;
+  let hotelDetails = { name: null, rating: null, reviewCount: null, address: null, image: null };
+  let browser = null;
+
+  // 1. Fallback: Nächte aus URL-Checkin/Checkout berechnen
+  const urlObj = new URL(url);
+  const checkin = urlObj.searchParams.get('checkin');
+  const checkout = urlObj.searchParams.get('checkout');
+  const adultsParam = parseInt(urlObj.searchParams.get('group_adults') || '0', 10);
+
+  let nightsFromUrl = null;
+  if (checkin && checkout) {
+    const diffTime = Math.abs(new Date(checkout) - new Date(checkin));
+    nightsFromUrl = Math.ceil(diffTime / (1000 * 60 * 60 * 24));
+  }
+
+  try {
+    browser = await chromium.launch({ headless: true });
+    const context = await browser.newContext({
+      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+      viewport: { width: 1280, height: 800 },
+      locale: 'de-DE',
+      timezoneId: 'Europe/Berlin'
+    });
+
+    const page = await context.newPage();
+    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 35000 });
+    await page.waitForTimeout(3000);
+
+    // 2. Hotel Details aus JSON-LD
+    hotelDetails = await page.evaluate(() => {
+      const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
+      for (const script of scripts) {
+        try {
+          const json = JSON.parse(script.innerText);
+          if (json['@type'] === 'Hotel' || json['@type'] === 'LodgingBusiness' || json.name) {
+            return {
+              name: json.name || null,
+              rating: json.aggregateRating?.ratingValue || null,
+              reviewCount: json.aggregateRating?.reviewCount || null,
+              address: json.address?.streetAddress || json.address?.addressLocality || null,
+              image: json.image || null
+            };
+          }
+        } catch (e) {}
+      }
+      return { name: null, rating: null, reviewCount: null, address: null, image: null };
+    });
+
+    // 3. Nächte, Personen & Zimmergröße direkt aus den Booking-DOM-Selektoren lesen
+    const domData = await page.evaluate(() => {
+      let dNights = null;
+      let dPersons = null;
+
+      // Gezielte Suche in den von dir genannten Selektoren
+      const targetSelectors = [
+        '.bui-price-display__label',
+        '#searchbox-horizontal-occupancy-label',
+        '[data-testid="price-for-x-nights"]',
+        '#group_recommendation .bui-price-display__label',
+        '[data-testid="occupancy-config"]'
+      ];
+
+      let combinedText = '';
+      for (const sel of targetSelectors) {
+        const els = document.querySelectorAll(sel);
+        els.forEach(el => {
+          if (el && el.innerText) combinedText += ' ' + el.innerText;
+        });
+      }
+
+      // Falls Selektoren leer sind, gesamten Text als Fallback nutzen
+      if (!combinedText.trim()) {
+        combinedText = document.body.innerText;
+      }
+
+      // Nächte matchen (z. B. "7 Nächte", "1 Nacht", "für 2 Nächte")
+      const nightMatch = combinedText.match(/(\d+)\s*Nächte?/i);
+      if (nightMatch) dNights = parseInt(nightMatch[1], 10);
+
+      // Personen matchen (z. B. "2 Erwachsene", "1 Erwachsener", "2 Gäste")
+      const personMatch = combinedText.match(/(\d+)\s*(?:Erwachsene|Erwachsener|Gäste|Personen)/i);
+      if (personMatch) dPersons = parseInt(personMatch[1], 10);
+
+      // Zimmergröße (m²)
+      const roomMatch = document.body.innerText.match(/(\d+)\s*m²/i);
+      const roomSize = roomMatch ? parseInt(roomMatch[1], 10) : null;
+
+      return { dNights, dPersons, roomSize };
+    });
+
+    roomSizeM2 = domData.roomSize;
+
+    // Finale Werte bestimmen (Priorität: URL -> DOM -> Standard-Fallback)
+    const finalNights = nightsFromUrl || domData.dNights || 1;
+    const finalPersons = adultsParam || domData.dPersons || 2;
+
+    // 4. Preis extrahieren
+    const priceSelectors = [
+      '[data-testid="price-and-discounted-price"]',
+      '.totalPrice .bui-price-display__value',
+      '.hprt-price-price'
+    ];
+
+    for (const selector of priceSelectors) {
+      const el = await page.$(selector);
+      if (el) {
+        const text = await el.innerText();
+        if (text && /\d/.test(text)) {
+          scrapedPrice = text.trim();
+          break;
+        }
+      }
+    }
+
+    // 5. Webhook mit exakten Daten absenden
+    if (webhookUrl && webhookUrl.startsWith('http')) {
+      await axios.post(webhookUrl, {
+        jobId: job.id,
+        url,
+        hotelName: hotelDetails.name,
+        price: scrapedPrice,
+        roomSizeM2,
+        rating: hotelDetails.rating,
+        reviewCount: hotelDetails.reviewCount,
+        address: hotelDetails.address,
+        image: hotelDetails.image,
+        nights: finalNights,
+        persons: finalPersons,
+        timestamp: new Date().toISOString()
+      });
+    }
+
+  } catch (error) {
+    console.error(`[Job ${job.id}] Fehler:`, error.message);
+  } finally {
+    if (browser) await browser.close();
+  }
+
+}, { ...REDIS_CONFIG, concurrency: 2 });