Files
trekkerv2/app/services/scraper.py
T

71 lines
2.6 KiB
Python

"""Trail guide web scrape: URL -> page text -> draft TrailGuide.
Uses httpx + BeautifulSoup. If an LLM is configured the text is run through
structured extraction; otherwise the user fills the review form from raw text.
"""
from __future__ import annotations
import logging
import httpx
from bs4 import BeautifulSoup
from app.models.trail_guide import TrailGuide, WaterCrossing
from app.services.llm import LLMService
log = logging.getLogger("trekker.scraper")
def fetch_page_text(url: str) -> tuple[str, str]:
"""Returns (title, readable_text)."""
resp = httpx.get(url, timeout=30, follow_redirects=True,
headers={"User-Agent": "Trekker/0.1 trail guide fetch"})
resp.raise_for_status()
soup = BeautifulSoup(resp.text, "html.parser")
for tag in soup(["script", "style", "nav", "footer", "header", "noscript"]):
tag.decompose()
title = soup.title.get_text(strip=True) if soup.title else url
text = "\n".join(
line for line in (ln.strip() for ln in soup.get_text("\n").splitlines()) if line
)
return title, text
def guide_from_extraction(extracted: dict, *, source: str, source_url: str | None = None,
pdf_path: str | None = None, raw_text: str | None = None,
fallback_name: str = "Unnamed trail guide") -> TrailGuide:
crossings = [
WaterCrossing(**wc) if isinstance(wc, dict) else WaterCrossing(notes=str(wc))
for wc in extracted.get("water_crossings") or []
]
return TrailGuide(
name=extracted.get("name") or fallback_name,
source=source,
source_url=source_url,
pdf_path=pdf_path,
difficulty=extracted.get("difficulty"),
obstacle_types=[str(o) for o in extracted.get("obstacle_types") or []],
water_crossings=crossings,
gate_codes=[str(g) for g in extracted.get("gate_codes") or []],
vehicle_recommendations=[str(v) for v in extracted.get("vehicle_recommendations") or []],
closures=[str(c) for c in extracted.get("closures") or []],
warnings=[str(w) for w in extracted.get("warnings") or []],
notes=extracted.get("notes"),
raw_text=raw_text,
)
def scrape_trail_guide(url: str, llm: LLMService) -> TrailGuide:
title, text = fetch_page_text(url)
extracted: dict = {}
if llm.configured and text:
try:
extracted = llm.extract_trail_data(text)
except Exception:
log.exception("LLM extraction failed for %s; returning raw-text draft", url)
return guide_from_extraction(
extracted, source="web", source_url=url,
raw_text=text[:50000], fallback_name=title,
)