437 lines
14 KiB
Plaintext
437 lines
14 KiB
Plaintext
========================================
|
|
PROJEKTSTRUKTUR
|
|
========================================
|
|
(Ermittelt via Git unter Berücksichtigung von .gitignore)
|
|
copyChat.sh
|
|
llm_context.txt
|
|
Dockerfile
|
|
README.md
|
|
docker-compose.yml
|
|
package.json
|
|
src/index.js
|
|
|
|
========================================
|
|
DATEIINHALTE
|
|
========================================
|
|
|
|
--- DATEI: Dockerfile ---
|
|
```
|
|
FROM mcr.microsoft.com/playwright:v1.45.0-jammy
|
|
|
|
WORKDIR /app
|
|
|
|
# Package-Dateien kopieren und installieren
|
|
COPY package*.json ./
|
|
RUN npm install
|
|
|
|
# Sourcecode kopieren
|
|
COPY . .
|
|
|
|
EXPOSE 3000
|
|
|
|
CMD ["node", "src/index.js"]
|
|
|
|
```
|
|
|
|
--- DATEI: README.md ---
|
|
```
|
|
|
|
```
|
|
|
|
--- DATEI: copyChat.sh ---
|
|
```
|
|
#!/usr/bin/env bash
|
|
|
|
OUTPUT_FILE="${1:-llm_context.txt}"
|
|
ROOT_DIR="${2:-.}"
|
|
|
|
# Absolute Pfade auflösen
|
|
OUTPUT_PATH="$(cd "$(dirname "$OUTPUT_FILE")" 2>/dev/null && pwd)/$(basename "$OUTPUT_FILE")"
|
|
|
|
cd "$ROOT_DIR" || exit 1
|
|
|
|
# Ausgabedatei initialisieren / leeren
|
|
> "$OUTPUT_PATH"
|
|
|
|
echo "========================================" >> "$OUTPUT_PATH"
|
|
echo "PROJEKTSTRUKTUR" >> "$OUTPUT_PATH"
|
|
echo "========================================" >> "$OUTPUT_PATH"
|
|
|
|
# Prüfen, ob Git verfügbar und das Verzeichnis ein Git-Repo ist
|
|
IS_GIT_REPO=false
|
|
if command -v git &> /dev/null && git rev-parse --is-inside-work-tree &> /dev/null; then
|
|
IS_GIT_REPO=true
|
|
fi
|
|
|
|
if [ "$IS_GIT_REPO" = true ]; then
|
|
echo "(Ermittelt via Git unter Berücksichtigung von .gitignore)" >> "$OUTPUT_PATH"
|
|
# Zeigt alle getrackten + ungetrackten Dateien an, ignoriert .gitignore-Treffer
|
|
git ls-files -co --exclude-standard >> "$OUTPUT_PATH"
|
|
else
|
|
if command -v tree &> /dev/null; then
|
|
tree -I 'node_modules|.git|venv|.venv|__pycache__|dist|build|.next|target' >> "$OUTPUT_PATH"
|
|
else
|
|
find . -maxdepth 3 -not -path '*/.*' -not -path '*/node_modules*' -not -path '*/venv*' >> "$OUTPUT_PATH"
|
|
fi
|
|
fi
|
|
|
|
echo -e "\n========================================" >> "$OUTPUT_PATH"
|
|
echo "DATEIINHALTE" >> "$OUTPUT_PATH"
|
|
echo "========================================" >> "$OUTPUT_PATH"
|
|
|
|
# Dateiliste ermitteln
|
|
get_file_list() {
|
|
if [ "$IS_GIT_REPO" = true ]; then
|
|
# -c = cached/tracked, -o = others/untracked, --exclude-standard = .gitignore beachten
|
|
git ls-files -co --exclude-standard
|
|
else
|
|
find . -type f \
|
|
-not -path '*/.git/*' \
|
|
-not -path '*/node_modules/*' \
|
|
-not -path '*/venv/*' \
|
|
-not -path '*/.venv/*' \
|
|
-not -path '*/__pycache__/*' \
|
|
-not -path '*/dist/*' \
|
|
-not -path '*/build/*' \
|
|
-not -path '*/.next/*'
|
|
fi
|
|
}
|
|
|
|
# Dateien verarbeiten
|
|
get_file_list | sort | while IFS= read -r file; do
|
|
# Überspringe leere Pfade, die Zieldatei selbst oder typische Binär-Endungen
|
|
[ -z "$file" ] && continue
|
|
[ "$file" = "$OUTPUT_FILE" ] && continue
|
|
[ "$(realpath "$file" 2>/dev/null)" = "$OUTPUT_PATH" ] && continue
|
|
|
|
case "$file" in
|
|
*.png|*.jpg|*.jpeg|*.gif|*.ico|*.pdf|*.zip|*.tar.gz|*.lock|package-lock.json|yarn.lock)
|
|
continue
|
|
;;
|
|
esac
|
|
|
|
# Prüfen, ob die Datei existiert und lesbarer Text ist
|
|
if [ -f "$file" ] && (grep -Iq . "$file" 2>/dev/null || [ ! -s "$file" ]); then
|
|
echo -e "\n--- DATEI: $file ---" >> "$OUTPUT_PATH"
|
|
echo '```' >> "$OUTPUT_PATH"
|
|
cat "$file" >> "$OUTPUT_PATH"
|
|
echo -e "\n"'```' >> "$OUTPUT_PATH"
|
|
fi
|
|
done
|
|
|
|
echo "Fertig! Projektkontext liegt in: $OUTPUT_PATH"
|
|
```
|
|
|
|
--- DATEI: docker-compose.yml ---
|
|
```
|
|
version: '3.8'
|
|
|
|
services:
|
|
redis:
|
|
image: redis:alpine
|
|
container_name: scraper-redis
|
|
restart: always
|
|
ports:
|
|
- "6379:6379"
|
|
|
|
scraper-app:
|
|
build: .
|
|
container_name: scraper-app
|
|
restart: always
|
|
ports:
|
|
- "3000:3000"
|
|
environment:
|
|
- REDIS_HOST=redis
|
|
- REDIS_PORT=6379
|
|
depends_on:
|
|
- redis
|
|
|
|
```
|
|
|
|
--- DATEI: package.json ---
|
|
```
|
|
{
|
|
"name": "scraper-service",
|
|
"version": "1.0.0",
|
|
"main": "src/index.js",
|
|
"type": "module",
|
|
"scripts": {
|
|
"start": "node src/index.js"
|
|
},
|
|
"dependencies": {
|
|
"crawlee": "^3.10.0",
|
|
"playwright": "1.45.0",
|
|
"express": "^4.19.0",
|
|
"bullmq": "^5.8.0",
|
|
"axios": "^1.7.0"
|
|
},
|
|
"overrides": {
|
|
"css-select": "5.1.0"
|
|
}
|
|
}
|
|
|
|
```
|
|
|
|
--- DATEI: src/index.js ---
|
|
```
|
|
import { Worker } from 'bullmq';
|
|
import { chromium } from 'playwright';
|
|
import axios from 'axios';
|
|
|
|
const REDIS_CONFIG = {
|
|
connection: {
|
|
host: process.env.REDIS_HOST || 'redis',
|
|
port: parseInt(process.env.REDIS_PORT || '6379', 10)
|
|
}
|
|
};
|
|
|
|
const WESTERN_COUNTRIES = [
|
|
'Deutschland', 'Schweiz', 'Österreich', 'Vereinigte Staaten', 'USA',
|
|
'Vereinigtes Königreich', 'Großbritannien', 'Niederlande', 'Frankreich',
|
|
'Spanien', 'Italien', 'Kanada', 'Australien', 'Belgien', 'Schweden',
|
|
'Norwegen', 'Dänemark', 'Finnland', 'Irland', 'Luxemburg', 'Indien'
|
|
];
|
|
|
|
const worker = new Worker('scrape-jobs', async (job) => {
|
|
const { url, webhookUrl, appWebhookUrl, hotelId } = job.data;
|
|
const isReviewScan = Boolean(webhookUrl && webhookUrl.startsWith('http'));
|
|
|
|
console.log(`[Job ${job.id}] Starte ${isReviewScan ? 'FULL-SCAN (Preis + KI-Reviews)' : 'PREIS-SCAN (Schnell)'} für: ${url}`);
|
|
|
|
let scrapedPrice = null;
|
|
let hasGenius = false;
|
|
let roomSizeM2 = null;
|
|
let hotelDetails = { name: null, rating: null, reviewCount: null, address: null, image: null };
|
|
let domData = { dNights: 1, dPersons: 2, roomSize: null, checkinDate: null, checkoutDate: null };
|
|
let extractedReviews = [];
|
|
let browser = null;
|
|
|
|
try {
|
|
browser = await chromium.launch({ headless: true });
|
|
const context = await browser.newContext({
|
|
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
|
|
viewport: { width: 1280, height: 900 },
|
|
locale: 'de-DE',
|
|
timezoneId: 'Europe/Berlin'
|
|
});
|
|
|
|
const page = await context.newPage();
|
|
let targetUrl = url;
|
|
if (isReviewScan && !targetUrl.includes('#tab-reviews')) {
|
|
targetUrl = targetUrl.replace('#tab-main', '') + '#tab-reviews';
|
|
}
|
|
|
|
await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 35000 });
|
|
await page.waitForTimeout(2500);
|
|
|
|
// DATUM AUS URL PARSEN
|
|
try {
|
|
const parsedUrl = new URL(url);
|
|
domData.checkinDate = parsedUrl.searchParams.get('checkin');
|
|
domData.checkoutDate = parsedUrl.searchParams.get('checkout');
|
|
} catch(e) {}
|
|
|
|
// 1. JSON-LD STAMMDATEN
|
|
hotelDetails = await page.evaluate(() => {
|
|
try {
|
|
const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
|
|
for (const script of scripts) {
|
|
try {
|
|
const json = JSON.parse(script.innerText || '{}');
|
|
if (json['@type'] === 'Hotel' || json['@type'] === 'LodgingBusiness' || json.name) {
|
|
return {
|
|
name: json.name || null,
|
|
rating: json.aggregateRating?.ratingValue || null,
|
|
reviewCount: json.aggregateRating?.reviewCount || null,
|
|
address: json.address?.streetAddress || json.address?.addressLocality || null,
|
|
image: Array.isArray(json.image) ? json.image[0] : (json.image || null)
|
|
};
|
|
}
|
|
} catch (e) {}
|
|
}
|
|
} catch (e) {}
|
|
return { name: null, rating: null, reviewCount: null, address: null, image: null };
|
|
});
|
|
|
|
// 2. DOM DATEN
|
|
const extraDom = await page.evaluate(() => {
|
|
try {
|
|
const bodyText = document.body ? (document.body.innerText || '') : '';
|
|
const nMatch = bodyText.match(/(\d+)\s*Nächte?/i);
|
|
const pMatch = bodyText.match(/(\d+)\s*(?:Erwachsene|Erwachsener|Gäste|Personen)/i);
|
|
const rmMatch = bodyText.match(/(\d+)\s*m²/i);
|
|
return {
|
|
dNights: nMatch ? parseInt(nMatch[1], 10) : 1,
|
|
dPersons: pMatch ? parseInt(pMatch[1], 10) : 2,
|
|
roomSize: rmMatch ? parseInt(rmMatch[1], 10) : null
|
|
};
|
|
} catch (e) {
|
|
return { dNights: 1, dPersons: 2, roomSize: null };
|
|
}
|
|
});
|
|
|
|
domData.dNights = extraDom.dNights;
|
|
domData.dPersons = extraDom.dPersons;
|
|
roomSizeM2 = extraDom.roomSize;
|
|
|
|
// PREIS AUSLESEN
|
|
scrapedPrice = await page.evaluate(() => {
|
|
const selectors = [
|
|
'[data-testid="price-and-discounted-price"]',
|
|
'.totalPrice .bui-price-display__value',
|
|
'.hprt-price-price',
|
|
'.prco-valign-middle-helper'
|
|
];
|
|
for (const sel of selectors) {
|
|
const el = document.querySelector(sel);
|
|
if (el && el.innerText && /\d/.test(el.innerText)) return el.innerText.trim();
|
|
}
|
|
return null;
|
|
});
|
|
|
|
// GENIUS BADGE ERKENNUNG (NEU)
|
|
hasGenius = await page.evaluate(() => {
|
|
const geniusSelectors = [
|
|
'.m-badge__genius',
|
|
'svg.genius-badge',
|
|
'.-genius-new_identity-genius_badge',
|
|
'.c-deals-container__genius-badge-fix',
|
|
'[aria-label*="Genius"]',
|
|
'[data-testid="deal-badge"]'
|
|
];
|
|
return geniusSelectors.some(sel => Boolean(document.querySelector(sel)));
|
|
});
|
|
|
|
if (hasGenius) {
|
|
console.log(`[Job ${job.id}] 💙 Genius-Badge erkannt (-10% Rabatt gilt)!`);
|
|
}
|
|
|
|
// 3. KOMMENTARE EXTRAHIEREN
|
|
if (isReviewScan) {
|
|
console.log(`[Job ${job.id}] Starte Review-Extraction mit Pagination...`);
|
|
try {
|
|
const reviewTab = await page.$('a[href*="#tab-reviews"], #js--hp-gallery-scorecard a, [data-testid="review-score-link"]');
|
|
if (reviewTab) {
|
|
await reviewTab.click();
|
|
await page.waitForTimeout(2000);
|
|
}
|
|
} catch (e) {}
|
|
|
|
await page.evaluate(() => window.scrollBy(0, 1000));
|
|
await page.waitForTimeout(2000);
|
|
|
|
let pageCount = 0;
|
|
const seenKeys = new Set();
|
|
|
|
while (extractedReviews.length < 80 && pageCount < 15) {
|
|
const pageReviews = await page.evaluate(({ westernCountries }) => {
|
|
const cards = Array.from(document.querySelectorAll('[data-testid="review-card"], [role="group"][aria-label*="Bewertung"], li.review_list_new_item_block, .review_item'));
|
|
const results = [];
|
|
const currentYear = 2026;
|
|
|
|
for (const card of cards) {
|
|
const cardText = card ? (card.innerText || '') : '';
|
|
if (!cardText) continue;
|
|
|
|
const isWestern = westernCountries.some(c => cardText.toLowerCase().includes(c.toLowerCase()));
|
|
if (!isWestern) continue;
|
|
|
|
const yearMatch = cardText.match(/\b(20\d{2})\b/);
|
|
if (yearMatch) {
|
|
const reviewYear = parseInt(yearMatch[1], 10);
|
|
if (currentYear - reviewYear > 2) continue;
|
|
}
|
|
|
|
const titleEl = card.querySelector('[data-testid="review-title"], h4');
|
|
const posEl = card.querySelector('[data-testid="review-positive-text"], .review_pos');
|
|
const negEl = card.querySelector('[data-testid="review-negative-text"], .review_neg');
|
|
const dateEl = card.querySelector('[data-testid="review-date"]');
|
|
|
|
const title = titleEl ? titleEl.innerText.trim() : '';
|
|
const positive = posEl ? posEl.innerText.trim() : '';
|
|
const negative = negEl ? negEl.innerText.trim() : '';
|
|
const dateText = dateEl ? dateEl.innerText.trim() : (yearMatch ? yearMatch[0] : 'Aktuell');
|
|
|
|
let fallbackText = '';
|
|
if (!positive && !negative) fallbackText = cardText.replace(title, '').substring(0, 300).trim();
|
|
|
|
if (title || positive || negative || fallbackText) {
|
|
results.push({ title, positive: positive || fallbackText, negative, date: dateText });
|
|
}
|
|
}
|
|
return results;
|
|
}, { westernCountries: WESTERN_COUNTRIES }).catch(() => []);
|
|
|
|
for (const rev of pageReviews) {
|
|
const key = `${rev.title}-${rev.positive}-${rev.date}`;
|
|
if (!seenKeys.has(key) && extractedReviews.length < 80) {
|
|
seenKeys.add(key);
|
|
extractedReviews.push(rev);
|
|
}
|
|
}
|
|
|
|
if (extractedReviews.length >= 80) break;
|
|
|
|
try {
|
|
const nextBtn = await page.$('#reviewCardsSection button:has(svg), button[aria-label*="Nächste"], button[aria-label*="Next"]');
|
|
if (nextBtn) {
|
|
const isDisabled = await nextBtn.getAttribute('disabled');
|
|
if (isDisabled !== null) break;
|
|
await nextBtn.click();
|
|
await page.waitForTimeout(2000);
|
|
pageCount++;
|
|
} else break;
|
|
} catch (e) { break; }
|
|
}
|
|
console.log(`[Job ${job.id}] ${extractedReviews.length} Kommentare extrahiert.`);
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error(`[Job ${job.id}] Fehler beim Scraping:`, error.message);
|
|
} finally {
|
|
if (browser) await browser.close();
|
|
|
|
// 1. PREISHISTORIE + DATUM SPEICHERN
|
|
if (appWebhookUrl) {
|
|
try {
|
|
await axios.post(appWebhookUrl, {
|
|
price: scrapedPrice,
|
|
hasGenius: hasGenius,
|
|
roomSizeM2,
|
|
hotelName: hotelDetails.name,
|
|
rating: hotelDetails.rating,
|
|
reviewCount: hotelDetails.reviewCount,
|
|
address: hotelDetails.address,
|
|
image: hotelDetails.image,
|
|
checkinDate: domData.checkinDate,
|
|
checkoutDate: domData.checkoutDate,
|
|
nights: domData.dNights || 1,
|
|
persons: domData.dPersons || 2,
|
|
});
|
|
console.log(`[Job ${job.id}] ✅ Preishistorie in Postgres gespeichert!`);
|
|
} catch (e) {
|
|
console.error(`[Job ${job.id}] ❌ Fehler bei Preishistorie:`, e.message);
|
|
}
|
|
}
|
|
|
|
// 2. REVIEWS AN N8N SENDEN
|
|
if (isReviewScan && extractedReviews.length > 0) {
|
|
try {
|
|
await axios.post(webhookUrl, {
|
|
jobId: job.id,
|
|
hotelId,
|
|
hotelName: hotelDetails.name,
|
|
reviews: extractedReviews,
|
|
timestamp: new Date().toISOString()
|
|
});
|
|
console.log(`[Job ${job.id}] 🚀 ${extractedReviews.length} Reviews an n8n gesendet!`);
|
|
} catch (e) {}
|
|
}
|
|
}
|
|
|
|
}, { ...REDIS_CONFIG, concurrency: 2 });
|
|
|
|
console.log('🚀 Scraper Worker bereit...');
|
|
|
|
```
|