Initial commit - Trekker V2 source
This commit is contained in:
commit
d71faf19e3
65 files changed
+3615
No files matched your search
@@ -0,0 +1,70 @@
|
||||
"""Trail guide web scrape: URL -> page text -> draft TrailGuide.
|
||||
|
||||
Uses httpx + BeautifulSoup. If an LLM is configured the text is run through
|
||||
structured extraction; otherwise the user fills the review form from raw text.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from app.models.trail_guide import TrailGuide, WaterCrossing
|
||||
from app.services.llm import LLMService
|
||||
|
||||
log = logging.getLogger("trekker.scraper")
|
||||
|
||||
|
||||
def fetch_page_text(url: str) -> tuple[str, str]:
|
||||
"""Returns (title, readable_text)."""
|
||||
resp = httpx.get(url, timeout=30, follow_redirects=True,
|
||||
headers={"User-Agent": "Trekker/0.1 trail guide fetch"})
|
||||
resp.raise_for_status()
|
||||
soup = BeautifulSoup(resp.text, "html.parser")
|
||||
for tag in soup(["script", "style", "nav", "footer", "header", "noscript"]):
|
||||
tag.decompose()
|
||||
title = soup.title.get_text(strip=True) if soup.title else url
|
||||
text = "\n".join(
|
||||
line for line in (ln.strip() for ln in soup.get_text("\n").splitlines()) if line
|
||||
)
|
||||
return title, text
|
||||
|
||||
|
||||
def guide_from_extraction(extracted: dict, *, source: str, source_url: str | None = None,
|
||||
pdf_path: str | None = None, raw_text: str | None = None,
|
||||
fallback_name: str = "Unnamed trail guide") -> TrailGuide:
|
||||
crossings = [
|
||||
WaterCrossing(**wc) if isinstance(wc, dict) else WaterCrossing(notes=str(wc))
|
||||
for wc in extracted.get("water_crossings") or []
|
||||
]
|
||||
return TrailGuide(
|
||||
name=extracted.get("name") or fallback_name,
|
||||
source=source,
|
||||
source_url=source_url,
|
||||
pdf_path=pdf_path,
|
||||
difficulty=extracted.get("difficulty"),
|
||||
obstacle_types=[str(o) for o in extracted.get("obstacle_types") or []],
|
||||
water_crossings=crossings,
|
||||
gate_codes=[str(g) for g in extracted.get("gate_codes") or []],
|
||||
vehicle_recommendations=[str(v) for v in extracted.get("vehicle_recommendations") or []],
|
||||
closures=[str(c) for c in extracted.get("closures") or []],
|
||||
warnings=[str(w) for w in extracted.get("warnings") or []],
|
||||
notes=extracted.get("notes"),
|
||||
raw_text=raw_text,
|
||||
)
|
||||
|
||||
|
||||
def scrape_trail_guide(url: str, llm: LLMService) -> TrailGuide:
|
||||
title, text = fetch_page_text(url)
|
||||
extracted: dict = {}
|
||||
if llm.configured and text:
|
||||
try:
|
||||
extracted = llm.extract_trail_data(text)
|
||||
except Exception:
|
||||
log.exception("LLM extraction failed for %s; returning raw-text draft", url)
|
||||
return guide_from_extraction(
|
||||
extracted, source="web", source_url=url,
|
||||
raw_text=text[:50000], fallback_name=title,
|
||||
)
|
||||
Reference in new issue
Block a user