Preishistory mit Genius
This commit is contained in:
436
llm_context.txt
Normal file
436
llm_context.txt
Normal file
@@ -0,0 +1,436 @@
|
||||
========================================
|
||||
PROJEKTSTRUKTUR
|
||||
========================================
|
||||
(Ermittelt via Git unter Berücksichtigung von .gitignore)
|
||||
copyChat.sh
|
||||
llm_context.txt
|
||||
Dockerfile
|
||||
README.md
|
||||
docker-compose.yml
|
||||
package.json
|
||||
src/index.js
|
||||
|
||||
========================================
|
||||
DATEIINHALTE
|
||||
========================================
|
||||
|
||||
--- DATEI: Dockerfile ---
|
||||
```
|
||||
FROM mcr.microsoft.com/playwright:v1.45.0-jammy
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Package-Dateien kopieren und installieren
|
||||
COPY package*.json ./
|
||||
RUN npm install
|
||||
|
||||
# Sourcecode kopieren
|
||||
COPY . .
|
||||
|
||||
EXPOSE 3000
|
||||
|
||||
CMD ["node", "src/index.js"]
|
||||
|
||||
```
|
||||
|
||||
--- DATEI: README.md ---
|
||||
```
|
||||
|
||||
```
|
||||
|
||||
--- DATEI: copyChat.sh ---
|
||||
```
|
||||
#!/usr/bin/env bash
|
||||
|
||||
OUTPUT_FILE="${1:-llm_context.txt}"
|
||||
ROOT_DIR="${2:-.}"
|
||||
|
||||
# Absolute Pfade auflösen
|
||||
OUTPUT_PATH="$(cd "$(dirname "$OUTPUT_FILE")" 2>/dev/null && pwd)/$(basename "$OUTPUT_FILE")"
|
||||
|
||||
cd "$ROOT_DIR" || exit 1
|
||||
|
||||
# Ausgabedatei initialisieren / leeren
|
||||
> "$OUTPUT_PATH"
|
||||
|
||||
echo "========================================" >> "$OUTPUT_PATH"
|
||||
echo "PROJEKTSTRUKTUR" >> "$OUTPUT_PATH"
|
||||
echo "========================================" >> "$OUTPUT_PATH"
|
||||
|
||||
# Prüfen, ob Git verfügbar und das Verzeichnis ein Git-Repo ist
|
||||
IS_GIT_REPO=false
|
||||
if command -v git &> /dev/null && git rev-parse --is-inside-work-tree &> /dev/null; then
|
||||
IS_GIT_REPO=true
|
||||
fi
|
||||
|
||||
if [ "$IS_GIT_REPO" = true ]; then
|
||||
echo "(Ermittelt via Git unter Berücksichtigung von .gitignore)" >> "$OUTPUT_PATH"
|
||||
# Zeigt alle getrackten + ungetrackten Dateien an, ignoriert .gitignore-Treffer
|
||||
git ls-files -co --exclude-standard >> "$OUTPUT_PATH"
|
||||
else
|
||||
if command -v tree &> /dev/null; then
|
||||
tree -I 'node_modules|.git|venv|.venv|__pycache__|dist|build|.next|target' >> "$OUTPUT_PATH"
|
||||
else
|
||||
find . -maxdepth 3 -not -path '*/.*' -not -path '*/node_modules*' -not -path '*/venv*' >> "$OUTPUT_PATH"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo -e "\n========================================" >> "$OUTPUT_PATH"
|
||||
echo "DATEIINHALTE" >> "$OUTPUT_PATH"
|
||||
echo "========================================" >> "$OUTPUT_PATH"
|
||||
|
||||
# Dateiliste ermitteln
|
||||
get_file_list() {
|
||||
if [ "$IS_GIT_REPO" = true ]; then
|
||||
# -c = cached/tracked, -o = others/untracked, --exclude-standard = .gitignore beachten
|
||||
git ls-files -co --exclude-standard
|
||||
else
|
||||
find . -type f \
|
||||
-not -path '*/.git/*' \
|
||||
-not -path '*/node_modules/*' \
|
||||
-not -path '*/venv/*' \
|
||||
-not -path '*/.venv/*' \
|
||||
-not -path '*/__pycache__/*' \
|
||||
-not -path '*/dist/*' \
|
||||
-not -path '*/build/*' \
|
||||
-not -path '*/.next/*'
|
||||
fi
|
||||
}
|
||||
|
||||
# Dateien verarbeiten
|
||||
get_file_list | sort | while IFS= read -r file; do
|
||||
# Überspringe leere Pfade, die Zieldatei selbst oder typische Binär-Endungen
|
||||
[ -z "$file" ] && continue
|
||||
[ "$file" = "$OUTPUT_FILE" ] && continue
|
||||
[ "$(realpath "$file" 2>/dev/null)" = "$OUTPUT_PATH" ] && continue
|
||||
|
||||
case "$file" in
|
||||
*.png|*.jpg|*.jpeg|*.gif|*.ico|*.pdf|*.zip|*.tar.gz|*.lock|package-lock.json|yarn.lock)
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
# Prüfen, ob die Datei existiert und lesbarer Text ist
|
||||
if [ -f "$file" ] && (grep -Iq . "$file" 2>/dev/null || [ ! -s "$file" ]); then
|
||||
echo -e "\n--- DATEI: $file ---" >> "$OUTPUT_PATH"
|
||||
echo '```' >> "$OUTPUT_PATH"
|
||||
cat "$file" >> "$OUTPUT_PATH"
|
||||
echo -e "\n"'```' >> "$OUTPUT_PATH"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Fertig! Projektkontext liegt in: $OUTPUT_PATH"
|
||||
```
|
||||
|
||||
--- DATEI: docker-compose.yml ---
|
||||
```
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
redis:
|
||||
image: redis:alpine
|
||||
container_name: scraper-redis
|
||||
restart: always
|
||||
ports:
|
||||
- "6379:6379"
|
||||
|
||||
scraper-app:
|
||||
build: .
|
||||
container_name: scraper-app
|
||||
restart: always
|
||||
ports:
|
||||
- "3000:3000"
|
||||
environment:
|
||||
- REDIS_HOST=redis
|
||||
- REDIS_PORT=6379
|
||||
depends_on:
|
||||
- redis
|
||||
|
||||
```
|
||||
|
||||
--- DATEI: package.json ---
|
||||
```
|
||||
{
|
||||
"name": "scraper-service",
|
||||
"version": "1.0.0",
|
||||
"main": "src/index.js",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "node src/index.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"crawlee": "^3.10.0",
|
||||
"playwright": "1.45.0",
|
||||
"express": "^4.19.0",
|
||||
"bullmq": "^5.8.0",
|
||||
"axios": "^1.7.0"
|
||||
},
|
||||
"overrides": {
|
||||
"css-select": "5.1.0"
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
--- DATEI: src/index.js ---
|
||||
```
|
||||
import { Worker } from 'bullmq';
|
||||
import { chromium } from 'playwright';
|
||||
import axios from 'axios';
|
||||
|
||||
const REDIS_CONFIG = {
|
||||
connection: {
|
||||
host: process.env.REDIS_HOST || 'redis',
|
||||
port: parseInt(process.env.REDIS_PORT || '6379', 10)
|
||||
}
|
||||
};
|
||||
|
||||
const WESTERN_COUNTRIES = [
|
||||
'Deutschland', 'Schweiz', 'Österreich', 'Vereinigte Staaten', 'USA',
|
||||
'Vereinigtes Königreich', 'Großbritannien', 'Niederlande', 'Frankreich',
|
||||
'Spanien', 'Italien', 'Kanada', 'Australien', 'Belgien', 'Schweden',
|
||||
'Norwegen', 'Dänemark', 'Finnland', 'Irland', 'Luxemburg', 'Indien'
|
||||
];
|
||||
|
||||
const worker = new Worker('scrape-jobs', async (job) => {
|
||||
const { url, webhookUrl, appWebhookUrl, hotelId } = job.data;
|
||||
const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http'));
|
||||
|
||||
console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews)' : 'PREIS-SCAN (Schnell)'} für: ${url}`);
|
||||
|
||||
let scrapedPrice = null;
|
||||
let hasGenius = false;
|
||||
let roomSizeM2 = null;
|
||||
let hotelDetails = { name: null, rating: null, reviewCount: null, address: null, image: null };
|
||||
let domData = { dNights: 1, dPersons: 2, roomSize: null, checkinDate: null, checkoutDate: null };
|
||||
let extractedReviews = [];
|
||||
let browser = null;
|
||||
|
||||
try {
|
||||
browser = await chromium.launch({ headless: true });
|
||||
const context = await browser.newContext({
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
|
||||
viewport: { width: 1280, height: 900 },
|
||||
locale: 'de-DE',
|
||||
timezoneId: 'Europe/Berlin'
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
let targetUrl = url;
|
||||
if (isReviewScan && !targetUrl.includes('#tab-reviews')) {
|
||||
targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews';
|
||||
}
|
||||
|
||||
await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 35000 });
|
||||
await page.waitForTimeout(2500);
|
||||
|
||||
// DATUM AUS URL PARSEN
|
||||
try {
|
||||
const parsedUrl = new URL(url);
|
||||
domData.checkinDate = parsedUrl.searchParams.get('checkin');
|
||||
domData.checkoutDate = parsedUrl.searchParams.get('checkout');
|
||||
} catch(e) {}
|
||||
|
||||
// 1. JSON-LD STAMMDATEN
|
||||
hotelDetails = await page.evaluate(() => {
|
||||
try {
|
||||
const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
|
||||
for (const script of scripts) {
|
||||
try {
|
||||
const json = JSON.parse(script.innerText || '{}');
|
||||
if (json['@type'] === 'Hotel' || json['@type'] === 'LodgingBusiness' || json.name) {
|
||||
return {
|
||||
name: json.name || null,
|
||||
rating: json.aggregateRating?.ratingValue || null,
|
||||
reviewCount: json.aggregateRating?.reviewCount || null,
|
||||
address: json.address?.streetAddress || json.address?.addressLocality || null,
|
||||
image: Array.isArray(json.image) ? json.image[0] : (json.image || null)
|
||||
};
|
||||
}
|
||||
} catch (e) {}
|
||||
}
|
||||
} catch (e) {}
|
||||
return { name: null, rating: null, reviewCount: null, address: null, image: null };
|
||||
});
|
||||
|
||||
// 2. DOM DATEN
|
||||
const extraDom = await page.evaluate(() => {
|
||||
try {
|
||||
const bodyText = document.body ? (document.body.innerText || '') : '';
|
||||
const nMatch = bodyText.match(/(\d+)\s*Nächte?/i);
|
||||
const pMatch = bodyText.match(/(\d+)\s*(?:Erwachsene|Erwachsener|Gäste|Personen)/i);
|
||||
const rmMatch = bodyText.match(/(\d+)\s*m²/i);
|
||||
return {
|
||||
dNights: nMatch ? parseInt(nMatch[1], 10) : 1,
|
||||
dPersons: pMatch ? parseInt(pMatch[1], 10) : 2,
|
||||
roomSize: rmMatch ? parseInt(rmMatch[1], 10) : null
|
||||
};
|
||||
} catch (e) {
|
||||
return { dNights: 1, dPersons: 2, roomSize: null };
|
||||
}
|
||||
});
|
||||
|
||||
domData.dNights = extraDom.dNights;
|
||||
domData.dPersons = extraDom.dPersons;
|
||||
roomSizeM2 = extraDom.roomSize;
|
||||
|
||||
// PREIS AUSLESEN
|
||||
scrapedPrice = await page.evaluate(() => {
|
||||
const selectors = [
|
||||
'[data-testid="price-and-discounted-price"]',
|
||||
'.totalPrice .bui-price-display__value',
|
||||
'.hprt-price-price',
|
||||
'.prco-valign-middle-helper'
|
||||
];
|
||||
for (const sel of selectors) {
|
||||
const el = document.querySelector(sel);
|
||||
if (el && el.innerText && /\d/.test(el.innerText)) return el.innerText.trim();
|
||||
}
|
||||
return null;
|
||||
});
|
||||
|
||||
// GENIUS BADGE ERKENNUNG (NEU)
|
||||
hasGenius = await page.evaluate(() => {
|
||||
const geniusSelectors = [
|
||||
'.m-badge__genius',
|
||||
'svg.genius-badge',
|
||||
'.-genius-new_identity-genius_badge',
|
||||
'.c-deals-container__genius-badge-fix',
|
||||
'[aria-label*="Genius"]',
|
||||
'[data-testid="deal-badge"]'
|
||||
];
|
||||
return geniusSelectors.some(sel => Boolean(document.querySelector(sel)));
|
||||
});
|
||||
|
||||
if (hasGenius) {
|
||||
console.log(`[Job ${job.id}] 💙 Genius-Badge erkannt (-10% Rabatt gilt)!`);
|
||||
}
|
||||
|
||||
// 3. KOMMENTARE EXTRAHIEREN
|
||||
if (isReviewScan) {
|
||||
console.log(`[Job ${job.id}] Starte Review-Extraction mit Pagination...`);
|
||||
try {
|
||||
const reviewTab = await page.$('a[href*="#tab-reviews"], #js--hp-gallery-scorecard a, [data-testid="review-score-link"]');
|
||||
if (reviewTab) {
|
||||
await reviewTab.click();
|
||||
await page.waitForTimeout(2000);
|
||||
}
|
||||
} catch (e) {}
|
||||
|
||||
await page.evaluate(() => window.scrollBy(0, 1000));
|
||||
await page.waitForTimeout(2000);
|
||||
|
||||
let pageCount = 0;
|
||||
const seenKeys = new Set();
|
||||
|
||||
while (extractedReviews.length < 80 && pageCount < 15) {
|
||||
const pageReviews = await page.evaluate(({ westernCountries }) => {
|
||||
const cards = Array.from(document.querySelectorAll('[data-testid="review-card"], [role="group"][aria-label*="Bewertung"], li.review_list_new_item_block, .review_item'));
|
||||
const results = [];
|
||||
const currentYear = 2026;
|
||||
|
||||
for (const card of cards) {
|
||||
const cardText = card ? (card.innerText || '') : '';
|
||||
if (!cardText) continue;
|
||||
|
||||
const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase()));
|
||||
if (!isWestern) continue;
|
||||
|
||||
const yearMatch = cardText.match(/\b(20\d{2})\b/);
|
||||
if (yearMatch) {
|
||||
const reviewYear = parseInt(yearMatch[1], 10);
|
||||
if (currentYear - reviewYear > 2) continue;
|
||||
}
|
||||
|
||||
const titleEl = card.querySelector('[data-testid="review-title"], h4');
|
||||
const posEl = card.querySelector('[data-testid="review-positive-text"], .review_pos');
|
||||
const negEl = card.querySelector('[data-testid="review-negative-text"], .review_neg');
|
||||
const dateEl = card.querySelector('[data-testid="review-date"]');
|
||||
|
||||
const title = titleEl ? titleEl.innerText.trim() : '';
|
||||
const positive = posEl ? posEl.innerText.trim() : '';
|
||||
const negative = negEl ? negEl.innerText.trim() : '';
|
||||
const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell');
|
||||
|
||||
let fallbackText = '';
|
||||
if (!positive && !negative) fallbackText = cardText.replace(title, '').substring(0, 300).trim();
|
||||
|
||||
if (title || positive || negative || fallbackText) {
|
||||
results.push({ title, positive: positive || fallbackText, negative, date: dateText });
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}, { westernCountries: WESTERN_COUNTRIES }).catch(() => []);
|
||||
|
||||
for (const rev of pageReviews) {
|
||||
const key = `${rev.title}-${rev.positive}-${rev.date}`;
|
||||
if (!seenKeys.has(key) && extractedReviews.length < 80) {
|
||||
seenKeys.add(key);
|
||||
extractedReviews.push(rev);
|
||||
}
|
||||
}
|
||||
|
||||
if (extractedReviews.length >= 80) break;
|
||||
|
||||
try {
|
||||
const nextBtn = await page.$('#reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"]');
|
||||
if (nextBtn) {
|
||||
const isDisabled = await nextBtn.getAttribute('disabled');
|
||||
if (isDisabled !== null) break;
|
||||
await nextBtn.click();
|
||||
await page.waitForTimeout(2000);
|
||||
pageCount++;
|
||||
} else break;
|
||||
} catch (e) { break; }
|
||||
}
|
||||
console.log(`[Job ${job.id}] ${extractedReviews.length} Kommentare extrahiert.`);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error(`[Job ${job.id}] Fehler beim Scraping:`, error.message);
|
||||
} finally {
|
||||
if (browser) await browser.close();
|
||||
|
||||
// 1. PREISHISTORIE + DATUM SPEICHERN
|
||||
if (appWebhookUrl) {
|
||||
try {
|
||||
await axios.post(appWebhookUrl, {
|
||||
price: scrapedPrice,
|
||||
hasGenius: hasGenius,
|
||||
roomSizeM2,
|
||||
hotelName: hotelDetails.name,
|
||||
rating: hotelDetails.rating,
|
||||
reviewCount: hotelDetails.reviewCount,
|
||||
address: hotelDetails.address,
|
||||
image: hotelDetails.image,
|
||||
checkinDate: domData.checkinDate,
|
||||
checkoutDate: domData.checkoutDate,
|
||||
nights: domData.dNights || 1,
|
||||
persons: domData.dPersons || 2,
|
||||
});
|
||||
console.log(`[Job ${job.id}] ✅ Preishistorie in Postgres gespeichert!`);
|
||||
} catch (e) {
|
||||
console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie:`, e.message);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. REVIEWS AN N8N SENDEN
|
||||
if (isReviewScan && extractedReviews.length > 0) {
|
||||
try {
|
||||
await axios.post(webhookUrl, {
|
||||
jobId: job.id,
|
||||
hotelId,
|
||||
hotelName: hotelDetails.name,
|
||||
reviews: extractedReviews,
|
||||
timestamp: new Date().toISOString()
|
||||
});
|
||||
console.log(`[Job ${job.id}] 🚀 ${extractedReviews.length} Reviews an n8n gesendet!`);
|
||||
} catch (e) {}
|
||||
}
|
||||
}
|
||||
|
||||
}, { ...REDIS_CONFIG, concurrency: 2 });
|
||||
|
||||
console.log('🚀 Scraper Worker bereit...');
|
||||
|
||||
```
|
||||
Reference in New Issue
Block a user