Preishistory mit Genius

src/index.js

@@ -18,13 +18,15 @@ const WESTERN_COUNTRIES = [
 
 const worker = new Worker('scrape-jobs', async (job) => {
   const { url, webhookUrl, appWebhookUrl, hotelId } = job.data;
-  console.log(`[Job ${job.id}] Starte Scraping für: ${url}`);
+  const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http'));
+
+  console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews)' : 'PREIS-SCAN (Schnell)'} für: ${url}`);
 
-  // Variablen auf Worker-Ebene deklarieren (Verhindert ReferenceError in finally)
   let scrapedPrice = null;
+  let hasGenius = false;
   let roomSizeM2 = null;
   let hotelDetails = { name: null, rating: null, reviewCount: null, address: null, image: null };
-  let domData = { dNights: 1, dPersons: 2, roomSize: null };
+  let domData = { dNights: 1, dPersons: 2, roomSize: null, checkinDate: null, checkoutDate: null };
   let extractedReviews = [];
   let browser = null;
 
@@ -38,212 +40,206 @@ const worker = new Worker('scrape-jobs', async (job) => {
     });
 
     const page = await context.newPage();
-
     let targetUrl = url;
-    if (!targetUrl.includes('#tab-reviews')) {
+    if (isReviewScan && !targetUrl.includes('#tab-reviews')) {
       targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews';
     }
-    if (!targetUrl.includes('sort_by=')) {
-      targetUrl += (targetUrl.includes('?') ? '&' : '?') + 'sort_by=f_recent_desc';
-    }
 
     await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 35000 });
     await page.waitForTimeout(2500);
 
-    // Klick auf Bewertungs-Reiter erzwingen
+    // DATUM AUS URL PARSEN
     try {
-      const reviewTab = await page.$('#js--hp-gallery-scorecard a, a[href*="#tab-reviews"], [data-testid="review-score-link"]');
-      if (reviewTab) {
-        await reviewTab.click();
-        await page.waitForTimeout(2000);
-      }
-    } catch (e) {}
+      const parsedUrl = new URL(url);
+      domData.checkinDate = parsedUrl.searchParams.get('checkin');
+      domData.checkoutDate = parsedUrl.searchParams.get('checkout');
+    } catch(e) {}
 
-    // Scrollen
-    await page.evaluate(() => {
-      const el = document.querySelector('#reviewListSorters') || document.querySelector('#tab-reviews');
-      if (el) el.scrollIntoView();
-      else window.scrollBy(0, 1000);
-    });
-    await page.waitForTimeout(2500);
-
-    // 1. JSON-LD Stammdaten
+    // 1. JSON-LD STAMMDATEN
     hotelDetails = await page.evaluate(() => {
-      const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
-      for (const script of scripts) {
-        try {
-          const json = JSON.parse(script.innerText);
-          if (json['@type'] === 'Hotel' || json['@type'] === 'LodgingBusiness' || json.name) {
-            return {
-              name: json.name || null,
-              rating: json.aggregateRating?.ratingValue || null,
-              reviewCount: json.aggregateRating?.reviewCount || null,
-              address: json.address?.streetAddress || json.address?.addressLocality || null,
-              image: json.image || null
-            };
-          }
-        } catch (e) {}
-      }
+      try {
+        const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
+        for (const script of scripts) {
+          try {
+            const json = JSON.parse(script.innerText || '{}');
+            if (json['@type'] === 'Hotel' || json['@type'] === 'LodgingBusiness' || json.name) {
+              return {
+                name: json.name || null,
+                rating: json.aggregateRating?.ratingValue || null,
+                reviewCount: json.aggregateRating?.reviewCount || null,
+                address: json.address?.streetAddress || json.address?.addressLocality || null,
+                image: Array.isArray(json.image) ? json.image[0] : (json.image || null)
+              };
+            }
+          } catch (e) {}
+        }
+      } catch (e) {}
       return { name: null, rating: null, reviewCount: null, address: null, image: null };
     });
 
-    // 2. Preis & DOM-Daten
-    domData = await page.evaluate(() => {
-      let dNights = null;
-      let dPersons = null;
-      const targetSelectors = ['.bui-price-display__label', '#searchbox-horizontal-occupancy-label', '[data-testid="price-for-x-nights"]'];
-      let combinedText = '';
-      targetSelectors.forEach(s => document.querySelectorAll(s).forEach(el => combinedText += ' ' + el.innerText));
-      if (!combinedText.trim()) combinedText = document.body.innerText;
-
-      const nMatch = combinedText.match(/(\d+)\s*Nächte?/i);
-      if (nMatch) dNights = parseInt(nMatch[1], 10);
-      const pMatch = combinedText.match(/(\d+)\s*(?:Erwachsene|Erwachsener|Gäste|Personen)/i);
-      if (pMatch) dPersons = parseInt(pMatch[1], 10);
-      const rmMatch = document.body.innerText.match(/(\d+)\s*m²/i);
-
-      return { dNights, dPersons, roomSize: rmMatch ? parseInt(rmMatch[1], 10) : null };
+    // 2. DOM DATEN
+    const extraDom = await page.evaluate(() => {
+      try {
+        const bodyText = document.body ? (document.body.innerText || '') : '';
+        const nMatch = bodyText.match(/(\d+)\s*Nächte?/i);
+        const pMatch = bodyText.match(/(\d+)\s*(?:Erwachsene|Erwachsener|Gäste|Personen)/i);
+        const rmMatch = bodyText.match(/(\d+)\s*m²/i);
+        return {
+          dNights: nMatch ? parseInt(nMatch[1], 10) : 1,
+          dPersons: pMatch ? parseInt(pMatch[1], 10) : 2,
+          roomSize: rmMatch ? parseInt(rmMatch[1], 10) : null
+        };
+      } catch (e) {
+        return { dNights: 1, dPersons: 2, roomSize: null };
+      }
     });
 
-    roomSizeM2 = domData.roomSize;
+    domData.dNights = extraDom.dNights;
+    domData.dPersons = extraDom.dPersons;
+    roomSizeM2 = extraDom.roomSize;
 
-    const priceSelectors = ['[data-testid="price-and-discounted-price"]', '.totalPrice .bui-price-display__value', '.hprt-price-price'];
-    for (const sel of priceSelectors) {
-      const el = await page.$(sel);
-      if (el) {
-        const text = await el.innerText();
-        if (text && /\d/.test(text)) { scrapedPrice = text.trim(); break; }
+    // PREIS AUSLESEN
+    scrapedPrice = await page.evaluate(() => {
+      const selectors = [
+        '[data-testid="price-and-discounted-price"]',
+        '.totalPrice .bui-price-display__value',
+        '.hprt-price-price',
+        '.prco-valign-middle-helper'
+      ];
+      for (const sel of selectors) {
+        const el = document.querySelector(sel);
+        if (el && el.innerText && /\d/.test(el.innerText)) return el.innerText.trim();
       }
+      return null;
+    });
+
+    // GENIUS BADGE ERKENNUNG (NEU)
+    hasGenius = await page.evaluate(() => {
+      const geniusSelectors = [
+        '.m-badge__genius',
+        'svg.genius-badge',
+        '.-genius-new_identity-genius_badge',
+        '.c-deals-container__genius-badge-fix',
+        '[aria-label*="Genius"]',
+        '[data-testid="deal-badge"]'
+      ];
+      return geniusSelectors.some(sel => Boolean(document.querySelector(sel)));
+    });
+
+    if (hasGenius) {
+      console.log(`[Job ${job.id}] 💙 Genius-Badge erkannt (-10% Rabatt gilt)!`);
     }
 
-    // 3. KOMMENTARE MIT PAGINATION BIS ZU 80 STÜCK EXTRAHIEREN
-    let pageCount = 0;
-    const seenKeys = new Set();
-
-    while (extractedReviews.length < 80 && pageCount < 15) {
-      const pageReviews = await page.evaluate(({ westernCountries }) => {
-        const dateElements = Array.from(document.querySelectorAll('[data-testid="review-date"]'));
-        const results = [];
-        const currentYear = 2026;
-
-        for (const dateEl of dateElements) {
-          // Navigiere hoch zum Container, der SOWOHL Person/Land ALS AUCH Bewertungstext enthält
-          let card = dateEl.parentElement;
-          while (card && card !== document.body) {
-            if (card.querySelector('[data-testid="review-avatar"]') || card.querySelector('[aria-label="Bewertende Person"]')) {
-              break;
-            }
-            card = card.parentElement;
-          }
-          if (!card || card === document.body) {
-            card = dateEl.closest('li') || dateEl.parentElement.parentElement.parentElement;
-          }
-
-          const cardText = card ? card.innerText : '';
-          if (!cardText) continue;
-
-          // Herkunftsland prüfen
-          const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase()));
-          if (!isWestern) continue;
-
-          // Datum prüfen (<= 2 Jahre)
-          const dateText = dateEl.innerText || '';
-          const yearMatch = cardText.match(/\b(20\d{2})\b/);
-          if (yearMatch) {
-            const reviewYear = parseInt(yearMatch[1], 10);
-            if (currentYear - reviewYear > 2) continue;
-          }
-
-          const titleEl = card.querySelector('[data-testid="review-title"]');
-          const posEl = card.querySelector('[data-testid="review-positive-text"], .review_pos');
-          const negEl = card.querySelector('[data-testid="review-negative-text"], .review_neg');
-
-          const title = titleEl ? titleEl.innerText.trim() : '';
-          const positive = posEl ? posEl.innerText.trim() : '';
-          const negative = negEl ? negEl.innerText.trim() : '';
-
-          let generalComment = '';
-          if (!positive && !negative) {
-            const commentEl = card.querySelector('.b99b6ef58f, .a17eaf5728');
-            if (commentEl && !commentEl.innerText.includes('keinen Kommentar')) {
-              generalComment = commentEl.innerText.trim();
-            }
-          }
-
-          if (title || positive || negative || generalComment) {
-            results.push({
-              title,
-              positive: positive || generalComment,
-              negative,
-              date: dateText
-            });
-          }
-        }
-
-        return results;
-      }, { westernCountries: WESTERN_COUNTRIES });
-
-      // Duplikate vermeiden
-      for (const rev of pageReviews) {
-        const key = `${rev.title}-${rev.positive}-${rev.date}`;
-        if (!seenKeys.has(key) && extractedReviews.length < 80) {
-          seenKeys.add(key);
-          extractedReviews.push(rev);
-        }
-      }
-
-      if (extractedReviews.length >= 80) break;
-
-      // Klick auf "Nächste Seite" Pfeil
+    // 3. KOMMENTARE EXTRAHIEREN
+    if (isReviewScan) {
+      console.log(`[Job ${job.id}] Starte Review-Extraction mit Pagination...`);
       try {
-        const nextBtn = await page.$(
-          '#reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"], [data-testid="pagination-next-page"]'
-        );
-
-        if (nextBtn) {
-          const isDisabled = await nextBtn.getAttribute('disabled');
-          if (isDisabled !== null) break;
-
-          await nextBtn.click();
-          await page.waitForTimeout(2500);
-          pageCount++;
-        } else {
-          break;
+        const reviewTab = await page.$('a[href*="#tab-reviews"], #js--hp-gallery-scorecard a, [data-testid="review-score-link"]');
+        if (reviewTab) {
+          await reviewTab.click();
+          await page.waitForTimeout(2000);
         }
-      } catch (e) {
-        break;
-      }
-    }
+      } catch (e) {}
 
-    console.log(`[Job ${job.id}] ${extractedReviews.length} Kommentare über ${pageCount + 1} Seiten extrahiert.`);
+      await page.evaluate(() => window.scrollBy(0, 1000));
+      await page.waitForTimeout(2000);
+
+      let pageCount = 0;
+      const seenKeys = new Set();
+
+      while (extractedReviews.length < 80 && pageCount < 15) {
+        const pageReviews = await page.evaluate(({ westernCountries }) => {
+          const cards = Array.from(document.querySelectorAll('[data-testid="review-card"], [role="group"][aria-label*="Bewertung"], li.review_list_new_item_block, .review_item'));
+          const results = [];
+          const currentYear = 2026;
+
+          for (const card of cards) {
+            const cardText = card ? (card.innerText || '') : '';
+            if (!cardText) continue;
+
+            const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase()));
+            if (!isWestern) continue;
+
+            const yearMatch = cardText.match(/\b(20\d{2})\b/);
+            if (yearMatch) {
+              const reviewYear = parseInt(yearMatch[1], 10);
+              if (currentYear - reviewYear > 2) continue;
+            }
+
+            const titleEl = card.querySelector('[data-testid="review-title"], h4');
+            const posEl = card.querySelector('[data-testid="review-positive-text"], .review_pos');
+            const negEl = card.querySelector('[data-testid="review-negative-text"], .review_neg');
+            const dateEl = card.querySelector('[data-testid="review-date"]');
+
+            const title = titleEl ? titleEl.innerText.trim() : '';
+            const positive = posEl ? posEl.innerText.trim() : '';
+            const negative = negEl ? negEl.innerText.trim() : '';
+            const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell');
+
+            let fallbackText = '';
+            if (!positive && !negative) fallbackText = cardText.replace(title, '').substring(0, 300).trim();
+
+            if (title || positive || negative || fallbackText) {
+              results.push({ title, positive: positive || fallbackText, negative, date: dateText });
+            }
+          }
+          return results;
+        }, { westernCountries: WESTERN_COUNTRIES }).catch(() => []);
+
+        for (const rev of pageReviews) {
+          const key = `${rev.title}-${rev.positive}-${rev.date}`;
+          if (!seenKeys.has(key) && extractedReviews.length < 80) {
+            seenKeys.add(key);
+            extractedReviews.push(rev);
+          }
+        }
+
+        if (extractedReviews.length >= 80) break;
+
+        try {
+          const nextBtn = await page.$('#reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"]');
+          if (nextBtn) {
+            const isDisabled = await nextBtn.getAttribute('disabled');
+            if (isDisabled !== null) break;
+            await nextBtn.click();
+            await page.waitForTimeout(2000);
+            pageCount++;
+          } else break;
+        } catch (e) { break; }
+      }
+      console.log(`[Job ${job.id}] ${extractedReviews.length} Kommentare extrahiert.`);
+    }
 
   } catch (error) {
     console.error(`[Job ${job.id}] Fehler beim Scraping:`, error.message);
   } finally {
     if (browser) await browser.close();
 
-    // 1. PREISHISTORIE IN POSTGRES SPEICHERN
+    // 1. PREISHISTORIE + DATUM SPEICHERN
     if (appWebhookUrl) {
       try {
         await axios.post(appWebhookUrl, {
           price: scrapedPrice,
+          hasGenius: hasGenius,
           roomSizeM2,
           hotelName: hotelDetails.name,
           rating: hotelDetails.rating,
           reviewCount: hotelDetails.reviewCount,
           address: hotelDetails.address,
           image: hotelDetails.image,
+          checkinDate: domData.checkinDate,
+          checkoutDate: domData.checkoutDate,
           nights: domData.dNights || 1,
           persons: domData.dPersons || 2,
         });
-        console.log(`[Job ${job.id}] ✅ Preishistorie direkt in Postgres gespeichert!`);
+        console.log(`[Job ${job.id}] ✅ Preishistorie in Postgres gespeichert!`);
       } catch (e) {
-        console.error(`[Job ${job.id}] ❌ Fehler beim Speichern der Preishistorie:`, e.message);
+        console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie:`, e.message);
       }
     }
 
     // 2. REVIEWS AN N8N SENDEN
-    if (webhookUrl) {
+    if (isReviewScan && extractedReviews.length > 0) {
       try {
         await axios.post(webhookUrl, {
           jobId: job.id,
@@ -253,9 +249,7 @@ const worker = new Worker('scrape-jobs', async (job) => {
           timestamp: new Date().toISOString()
         });
         console.log(`[Job ${job.id}] 🚀 ${extractedReviews.length} Reviews an n8n gesendet!`);
-      } catch (e) {
-        console.error(`[Job ${job.id}] ❌ Fehler beim Senden an n8n:`, e.message);
-      }
+      } catch (e) {}
     }
   }