71 lines
2.6 KiB
Python
71 lines
2.6 KiB
Python
"""Trail guide web scrape: URL -> page text -> draft TrailGuide.
|
|
|
|
Uses httpx + BeautifulSoup. If an LLM is configured the text is run through
|
|
structured extraction; otherwise the user fills the review form from raw text.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
|
|
from app.models.trail_guide import TrailGuide, WaterCrossing
|
|
from app.services.llm import LLMService
|
|
|
|
log = logging.getLogger("trekker.scraper")
|
|
|
|
|
|
def fetch_page_text(url: str) -> tuple[str, str]:
|
|
"""Returns (title, readable_text)."""
|
|
resp = httpx.get(url, timeout=30, follow_redirects=True,
|
|
headers={"User-Agent": "Trekker/0.1 trail guide fetch"})
|
|
resp.raise_for_status()
|
|
soup = BeautifulSoup(resp.text, "html.parser")
|
|
for tag in soup(["script", "style", "nav", "footer", "header", "noscript"]):
|
|
tag.decompose()
|
|
title = soup.title.get_text(strip=True) if soup.title else url
|
|
text = "\n".join(
|
|
line for line in (ln.strip() for ln in soup.get_text("\n").splitlines()) if line
|
|
)
|
|
return title, text
|
|
|
|
|
|
def guide_from_extraction(extracted: dict, *, source: str, source_url: str | None = None,
|
|
pdf_path: str | None = None, raw_text: str | None = None,
|
|
fallback_name: str = "Unnamed trail guide") -> TrailGuide:
|
|
crossings = [
|
|
WaterCrossing(**wc) if isinstance(wc, dict) else WaterCrossing(notes=str(wc))
|
|
for wc in extracted.get("water_crossings") or []
|
|
]
|
|
return TrailGuide(
|
|
name=extracted.get("name") or fallback_name,
|
|
source=source,
|
|
source_url=source_url,
|
|
pdf_path=pdf_path,
|
|
difficulty=extracted.get("difficulty"),
|
|
obstacle_types=[str(o) for o in extracted.get("obstacle_types") or []],
|
|
water_crossings=crossings,
|
|
gate_codes=[str(g) for g in extracted.get("gate_codes") or []],
|
|
vehicle_recommendations=[str(v) for v in extracted.get("vehicle_recommendations") or []],
|
|
closures=[str(c) for c in extracted.get("closures") or []],
|
|
warnings=[str(w) for w in extracted.get("warnings") or []],
|
|
notes=extracted.get("notes"),
|
|
raw_text=raw_text,
|
|
)
|
|
|
|
|
|
def scrape_trail_guide(url: str, llm: LLMService) -> TrailGuide:
|
|
title, text = fetch_page_text(url)
|
|
extracted: dict = {}
|
|
if llm.configured and text:
|
|
try:
|
|
extracted = llm.extract_trail_data(text)
|
|
except Exception:
|
|
log.exception("LLM extraction failed for %s; returning raw-text draft", url)
|
|
return guide_from_extraction(
|
|
extracted, source="web", source_url=url,
|
|
raw_text=text[:50000], fallback_name=title,
|
|
)
|