next env

AGENTS.md

@@ -0,0 +1,21 @@
+# Engineering & Scraping Guidelines
+
+## 1. Git Commit & Repository Hygiene
+- **Conventional Commits:** Write clear, concise commit messages prefixed with standard types (e.g., `feat:`, `fix:`, `refactor:`, `docs:`).
+- **Exclude Transient Artifacts:** Never stage or commit generated debug files (e.g., `src/*.html`, `src/*.png`, large logs) to version control. Always maintain `.gitignore` to keep git packs lean and prevent remote unpacker errors on Gitea/GitHub.
+
+## 2. Web Scraping & DOM Inspection Strategy
+- **Rendered DOM Dumping for Debugging:** When debugging extraction failures on dynamic or anti-bot protected pages:
+  - Capture the post-render HTML via `await page.content()` and save it to `src/hotel-debug.html`.
+  - Capture a synchronized screenshot (`src/hotel-debug.png`).
+  - Inspect the raw rendered DOM to verify if the page was served as standard SSR, an A/B test SPA variant, or intercepted by a challenge (e.g. AWS WAF).
+- **Multi-Tier Extraction Hierarchy:** Always implement fallbacks in this order:
+  1. **Deep JSON-LD Parsing:** Traverse all `<script type="application/ld+json">` tags, including `@graph` arrays and varied accommodation schemas (`Hotel`, `Resort`, `Apartment`, `Hostel`, `LodgingBusiness`, etc.).
+  2. **DOM Selectors:** Fallback to standard test IDs, classes, and element hierarchy (`data-testid`, header titles, score components).
+  3. **Meta Tags:** Fallback to OpenGraph and Twitter meta tags (`og:title`, `og:image`, `og:description`).
+  4. **URL Slug Extraction:** As an ultimate safety net, derive clean human-readable names from the URL path if the HTML body was obstructed.
+- **Anti-Bot & WAF Handling:**
+  - Sanitize input URLs by stripping transient tokens (`chal_t`, `force_referer`, session IDs).
+  - Emulate full browser properties (`window.chrome`, `navigator.plugins`, `navigator.languages`).
+  - Actively detect WAF challenge interstitial pages and wait for client-side JavaScript token computation and automatic redirection before evaluating the DOM.
+

next-env.d.ts

@@ -0,0 +1,5 @@
+/// <reference types="next" />
+/// <reference types="next/image-types/global" />
+
+// NOTE: This file should not be edited
+// see https://nextjs.org/docs/app/building-your-application/configuring/typescript for more information.

src/app/api/hotels/bulk/route.ts

@@ -0,0 +1,113 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { query } from '@/lib/db';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+export async function POST(req: NextRequest) {
+  try {
+    const { urls, bulk_text, trip_id } = await req.json();
+
+    let rawUrls: string[] = [];
+    if (Array.isArray(urls)) {
+      rawUrls = urls;
+    } else if (typeof bulk_text === 'string') {
+      rawUrls = bulk_text
+        .split(/[\r\n,\s]+/)
+        .map(u => u.trim())
+        .filter(u => u.length > 0);
+    }
+
+    // Filter valid Booking.com URLs
+    const validUrls = Array.from(
+      new Set(
+        rawUrls.filter(u => {
+          try {
+            const parsed = new URL(u);
+            return parsed.hostname.includes('booking.com') && parsed.pathname.includes('/hotel/');
+          } catch {
+            return false;
+          }
+        })
+      )
+    );
+
+    if (validUrls.length === 0) {
+      return NextResponse.json(
+        { success: false, error: 'Keine gültigen Booking.com Hotel-Links gefunden.' },
+        { status: 400 }
+      );
+    }
+
+    const insertedHotels = [];
+    const bullmq = typeof window === 'undefined' ? eval('require')('bullmq') : null;
+    let scrapeQueue: any = null;
+
+    if (bullmq && bullmq.Queue) {
+      const { Queue } = bullmq;
+      const connection = {
+        host: process.env.REDIS_HOST || '172.17.0.1',
+        port: parseInt(process.env.REDIS_PORT || '6379', 10),
+        connectTimeout: 5000,
+        maxRetriesPerRequest: 3,
+      };
+      scrapeQueue = new Queue('scrape-jobs', { connection });
+    }
+
+    for (const bookingUrl of validUrls) {
+      let checkin_date: string | null = null;
+      let checkout_date: string | null = null;
+      let hotelName = 'Neues Hotel';
+
+      try {
+        const parsed = new URL(bookingUrl);
+        checkin_date = parsed.searchParams.get('checkin');
+        checkout_date = parsed.searchParams.get('checkout');
+        const slugMatch = parsed.pathname.match(/\/hotel\/[a-z]{2}\/([^.]+)/i);
+        if (slugMatch && slugMatch[1]) {
+          hotelName = slugMatch[1]
+            .replace(/-/g, ' ')
+            .replace(/\b\w/g, c => c.toUpperCase());
+        }
+      } catch {}
+
+      const res = await query(
+        `INSERT INTO hotels (name, booking_url, checkin_date, checkout_date, trip_id, is_active)
+         VALUES ($1, $2, $3, $4, $5, true)
+         RETURNING *`,
+        [hotelName, bookingUrl, checkin_date, checkout_date, trip_id || null]
+      );
+
+      const newHotel = res.rows[0];
+      insertedHotels.push(newHotel);
+
+      if (scrapeQueue) {
+        try {
+          const appWebhookUrl = `http://172.17.0.1:3002/api/webhooks/scraper?hotelId=${newHotel.id}`;
+          await scrapeQueue.add('scrape-hotel', {
+            hotelId: newHotel.id,
+            url: newHotel.booking_url,
+            appWebhookUrl,
+            webhookUrl: null,
+          });
+        } catch (queueErr: any) {
+          console.error(`Fehler beim Einreihen von Hotel ${newHotel.id}:`, queueErr.message);
+        }
+      }
+    }
+
+    if (scrapeQueue) {
+      await scrapeQueue.close().catch(() => {});
+    }
+
+    return NextResponse.json({
+      success: true,
+      message: `${insertedHotels.length} Hotel(s) erfolgreich importiert und zur Erfassung eingereiht!`,
+      count: insertedHotels.length,
+      hotels: insertedHotels,
+    });
+  } catch (error: any) {
+    console.error('Fehler in POST /api/hotels/bulk:', error.message);
+    return NextResponse.json({ success: false, error: error.message }, { status: 500 });
+  }
+}

src/app/api/telegram/test/route.ts

@@ -0,0 +1,46 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { sendTelegramMessage } from '@/lib/telegram';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+export async function POST(req: NextRequest) {
+  try {
+    const botToken = process.env.TELEGRAM_BOT_TOKEN;
+    const chatId = process.env.TELEGRAM_CHAT_ID;
+
+    if (!botToken || !chatId) {
+      return NextResponse.json({
+        success: false,
+        error: 'TELEGRAM_BOT_TOKEN oder TELEGRAM_CHAT_ID ist in .env nicht gesetzt.',
+        configured: false,
+      }, { status: 400 });
+    }
+
+    const testMessage = `🤖 <b>Telegram-Benachrichtigungstest</b>
+
+✅ Die Verbindung zwischen deiner <b>Travel Tracker App</b> und dem Telegram Bot funktioniert einwandfrei!
+
+🔔 Sobald ein Hotelpreis sinkt oder ein neuer Bestpreis erreicht wird, erhältst du hier automatisch Push-Nachrichten.`;
+
+    const res = await sendTelegramMessage(testMessage);
+
+    if (res.success) {
+      return NextResponse.json({
+        success: true,
+        message: 'Test-Nachricht wurde erfolgreich an Telegram gesendet!',
+        chatId: chatId.slice(0, 4) + '***',
+      });
+    } else {
+      return NextResponse.json({
+        success: false,
+        error: res.error || 'Fehler beim Senden an Telegram API',
+      }, { status: 500 });
+    }
+  } catch (error: any) {
+    return NextResponse.json({
+      success: false,
+      error: error.message,
+    }, { status: 500 });
+  }
+}

src/app/api/webhooks/scraper/route.ts

@@ -1,5 +1,6 @@
 import { NextRequest, NextResponse } from 'next/server';
 import { query } from '@/lib/db';
+import { notifyPriceChange } from '@/lib/telegram';
 
 function parseGermanPrice(priceStr: any): number | null {
   if (!priceStr) return null;
@@ -42,11 +43,52 @@ export async function POST(req: NextRequest) {
 
     if (numericPrice !== null && numericPrice > 0) {
       try {
+        // Vorherigen Preis und bisherigen Allzeit-Tiefstpreis abfragen
+        const prevRes = await query(
+          `SELECT 
+             h.name, h.booking_url, h.checkin_date, h.checkout_date,
+             t.name as trip_name,
+             (SELECT price_numeric FROM hotel_price_history WHERE hotel_id = h.id ORDER BY tracked_at DESC, id DESC LIMIT 1) as last_price,
+             (SELECT MIN(price_numeric) FROM hotel_price_history WHERE hotel_id = h.id AND price_numeric > 0) as min_price
+           FROM hotels h
+           LEFT JOIN trips t ON h.trip_id = t.id
+           WHERE h.id = $1::uuid`,
+          [hotelId]
+        );
+
+        const hotelRecord = prevRes.rows[0] || null;
+        const previousPrice = hotelRecord?.last_price ? Math.round(Number(hotelRecord.last_price)) : null;
+        const allTimeBestPrice = hotelRecord?.min_price ? Math.round(Number(hotelRecord.min_price)) : null;
+
         await query(
           `INSERT INTO hotel_price_history (hotel_id, price_numeric, genius_price_numeric, price_raw, currency, nights, persons, tracked_at)
-       VALUES ($1::uuid, $2, $3, $4, 'EUR', $5, $6, NOW())`,
+           VALUES ($1::uuid, $2, $3, $4, 'EUR', $5, $6, NOW())`,
           [hotelId, numericPrice, geniusPriceNumeric, String(body.price || ''), body.nights || 1, body.persons || 2]
         );
+
+        // Telegram Push-Benachrichtigung bei Preissenkung oder neuem Allzeit-Bestpreis
+        if (hotelRecord && previousPrice !== null && previousPrice > 0) {
+          const isPriceDrop = numericPrice < previousPrice;
+          const isNewBestPrice = allTimeBestPrice !== null ? (numericPrice < allTimeBestPrice) : true;
+
+          if (isPriceDrop || isNewBestPrice) {
+            notifyPriceChange({
+              hotelName: hotelRecord.name || body.hotelName || 'Hotel',
+              tripName: hotelRecord.trip_name || null,
+              oldPrice: previousPrice,
+              newPrice: numericPrice,
+              newGeniusPrice: geniusPriceNumeric,
+              allTimeBestPrice: allTimeBestPrice,
+              isNewBestPrice,
+              isPriceDrop,
+              checkinDate: hotelRecord.checkin_date || body.checkin || null,
+              checkoutDate: hotelRecord.checkout_date || body.checkout || null,
+              nights: body.nights || null,
+              persons: body.persons || null,
+              bookingUrl: hotelRecord.booking_url || body.url || ''
+            }).catch(e => console.error('[Telegram] Fehler beim Senden der Benachrichtigung:', e.message));
+          }
+        }
       } catch (err: any) {
         console.error('DB Insert Error (hotel_price_history):', err.message);
       }

src/lib/telegram.ts

@@ -0,0 +1,111 @@
+export interface PriceNotificationPayload {
+  hotelName: string;
+  tripName?: string | null;
+  oldPrice: number | null;
+  newPrice: number;
+  newGeniusPrice?: number | null;
+  allTimeBestPrice?: number | null;
+  isNewBestPrice: boolean;
+  isPriceDrop: boolean;
+  checkinDate?: string | null;
+  checkoutDate?: string | null;
+  nights?: number | null;
+  persons?: number | null;
+  bookingUrl: string;
+}
+
+export async function sendTelegramMessage(htmlText: string): Promise<{ success: boolean; error?: string }> {
+  const botToken = process.env.TELEGRAM_BOT_TOKEN;
+  const chatId = process.env.TELEGRAM_CHAT_ID;
+
+  if (!botToken || !chatId) {
+    console.log('[Telegram] Keine TELEGRAM_BOT_TOKEN oder TELEGRAM_CHAT_ID in .env gesetzt – überspringe Benachrichtigung.');
+    return { success: false, error: 'Telegram Bot Token oder Chat ID nicht konfiguriert.' };
+  }
+
+  try {
+    const url = `https://api.telegram.org/bot${botToken}/sendMessage`;
+    const response = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        chat_id: chatId,
+        text: htmlText,
+        parse_mode: 'HTML',
+        disable_web_page_preview: false,
+      }),
+    });
+
+    const data = await response.json();
+    if (!data.ok) {
+      console.error('[Telegram] API Fehler:', data.description);
+      return { success: false, error: data.description };
+    }
+
+    console.log('[Telegram] ✅ Nachricht erfolgreich versendet!');
+    return { success: true };
+  } catch (err: any) {
+    console.error('[Telegram] Netzwerk- oder Request-Fehler:', err.message);
+    return { success: false, error: err.message };
+  }
+}
+
+export async function notifyPriceChange(payload: PriceNotificationPayload): Promise<{ success: boolean; error?: string }> {
+  const {
+    hotelName,
+    tripName,
+    oldPrice,
+    newPrice,
+    newGeniusPrice,
+    isNewBestPrice,
+    isPriceDrop,
+    checkinDate,
+    checkoutDate,
+    nights,
+    persons,
+    bookingUrl
+  } = payload;
+
+  let badge = '📉 <b>PREIS GESUNKEN!</b>';
+  if (isNewBestPrice) {
+    badge = '🔥 <b>NEUER ALLZEIT-TIEFSTPREIS!</b>';
+  }
+
+  let deltaStr = '';
+  if (oldPrice !== null && oldPrice > 0 && oldPrice !== newPrice) {
+    const diff = newPrice - oldPrice;
+    const pct = Math.round(((oldPrice - newPrice) / oldPrice) * 100);
+    deltaStr = `\n📉 <b>Ersparnis:</b> ${Math.abs(diff)} € (${pct}% günstiger! Vorher: ${oldPrice} €)`;
+  }
+
+  let geniusStr = '';
+  if (newGeniusPrice && newGeniusPrice > 0) {
+    geniusStr = `\n💙 <b>Genius-Preis (-10%):</b> <b>${newGeniusPrice} €</b>`;
+  }
+
+  let dateStr = '';
+  if (checkinDate && checkoutDate) {
+    const formatDate = (d: string) => {
+      try {
+        return new Date(d).toLocaleDateString('de-DE', { day: '2-digit', month: '2-digit', year: 'numeric' });
+      } catch {
+        return d;
+      }
+    };
+    dateStr = `\n🗓️ <b>Reisezeitraum:</b> ${formatDate(checkinDate)} – ${formatDate(checkoutDate)}`;
+    if (nights) {
+      dateStr += ` (${nights} ${nights === 1 ? 'Nacht' : 'Nächte'}${persons ? ` · ${persons} Pers.` : ''})`;
+    }
+  }
+
+  const tripStr = tripName ? `\n🌍 <b>Reise:</b> ${tripName}` : '';
+
+  const message = `${badge}
+
+🏨 <b>${hotelName}</b>${tripStr}
+💰 <b>Neuer Preis:</b> <b>${newPrice} €</b>${geniusStr}${deltaStr}${dateStr}
+
+<a href="${bookingUrl}">👉 Direkt zu Booking.com öffnen ↗</a>`;
+
+  return sendTelegramMessage(message);
+}