"""Trail guide web scrape: URL -> page text -> draft TrailGuide. Uses httpx + BeautifulSoup. If an LLM is configured the text is run through structured extraction; otherwise the user fills the review form from raw text. """ from __future__ import annotations import logging import httpx from bs4 import BeautifulSoup from app.models.trail_guide import TrailGuide, WaterCrossing from app.services.llm import LLMService log = logging.getLogger("trekker.scraper") def fetch_page_text(url: str) -> tuple[str, str]: """Returns (title, readable_text).""" resp = httpx.get(url, timeout=30, follow_redirects=True, headers={"User-Agent": "Trekker/0.1 trail guide fetch"}) resp.raise_for_status() soup = BeautifulSoup(resp.text, "html.parser") for tag in soup(["script", "style", "nav", "footer", "header", "noscript"]): tag.decompose() title = soup.title.get_text(strip=True) if soup.title else url text = "\n".join( line for line in (ln.strip() for ln in soup.get_text("\n").splitlines()) if line ) return title, text def guide_from_extraction(extracted: dict, *, source: str, source_url: str | None = None, pdf_path: str | None = None, raw_text: str | None = None, fallback_name: str = "Unnamed trail guide") -> TrailGuide: crossings = [ WaterCrossing(**wc) if isinstance(wc, dict) else WaterCrossing(notes=str(wc)) for wc in extracted.get("water_crossings") or [] ] return TrailGuide( name=extracted.get("name") or fallback_name, source=source, source_url=source_url, pdf_path=pdf_path, difficulty=extracted.get("difficulty"), obstacle_types=[str(o) for o in extracted.get("obstacle_types") or []], water_crossings=crossings, gate_codes=[str(g) for g in extracted.get("gate_codes") or []], vehicle_recommendations=[str(v) for v in extracted.get("vehicle_recommendations") or []], closures=[str(c) for c in extracted.get("closures") or []], warnings=[str(w) for w in extracted.get("warnings") or []], notes=extracted.get("notes"), raw_text=raw_text, ) def scrape_trail_guide(url: str, llm: LLMService) -> TrailGuide: title, text = fetch_page_text(url) extracted: dict = {} if llm.configured and text: try: extracted = llm.extract_trail_data(text) except Exception: log.exception("LLM extraction failed for %s; returning raw-text draft", url) return guide_from_extraction( extracted, source="web", source_url=url, raw_text=text[:50000], fallback_name=title, )