Initial commit - Trekker V2 source
This commit is contained in:
commit
d71faf19e3
65 files changed
+3615
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,82 @@
|
||||
"""Elevation lookup from Terrarium-encoded terrain tiles (AWS Open Data).
|
||||
|
||||
decode: elevation_m = (R * 256 + G + B / 256) - 32768
|
||||
Tiles are cached on disk under data_dir/terrain so offline packs and the
|
||||
segmenter share the same cache.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import logging
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
from PIL import Image
|
||||
|
||||
from app.config import settings
|
||||
from app.models.route import TrkPt
|
||||
|
||||
log = logging.getLogger("trekker.elevation")
|
||||
|
||||
TILE_SIZE = 256
|
||||
DEFAULT_ZOOM = 12
|
||||
|
||||
|
||||
def lonlat_to_tile(lon: float, lat: float, zoom: int) -> tuple[int, int, float, float]:
|
||||
"""Tile x/y plus fractional pixel offset within the tile."""
|
||||
n = 2**zoom
|
||||
xf = (lon + 180.0) / 360.0 * n
|
||||
lat_r = math.radians(lat)
|
||||
yf = (1.0 - math.asinh(math.tan(lat_r)) / math.pi) / 2.0 * n
|
||||
x, y = int(xf), int(yf)
|
||||
return x, y, (xf - x) * TILE_SIZE, (yf - y) * TILE_SIZE
|
||||
|
||||
|
||||
class ElevationService:
|
||||
def __init__(self, cache_dir: str | None = None, terrain_url: str | None = None):
|
||||
self.terrain_url = terrain_url or settings.tiles.terrain_url
|
||||
self.cache_dir = Path(cache_dir or Path(settings.data_dir) / "terrain")
|
||||
self._mem: dict[tuple[int, int, int], Image.Image] = {}
|
||||
|
||||
def _tile(self, z: int, x: int, y: int) -> Image.Image | None:
|
||||
key = (z, x, y)
|
||||
if key in self._mem:
|
||||
return self._mem[key]
|
||||
path = self.cache_dir / str(z) / str(x) / f"{y}.png"
|
||||
if not path.is_file():
|
||||
url = self.terrain_url.format(z=z, x=x, y=y)
|
||||
try:
|
||||
resp = httpx.get(url, timeout=30,
|
||||
headers={"User-Agent": settings.tiles.user_agent})
|
||||
resp.raise_for_status()
|
||||
except httpx.HTTPError as e:
|
||||
log.warning("terrain tile %s/%s/%s fetch failed: %s", z, x, y, e)
|
||||
return None
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(resp.content)
|
||||
img = Image.open(io.BytesIO(path.read_bytes())).convert("RGB")
|
||||
if len(self._mem) > 64:
|
||||
self._mem.clear()
|
||||
self._mem[key] = img
|
||||
return img
|
||||
|
||||
def elevation_at(self, lat: float, lon: float, zoom: int = DEFAULT_ZOOM) -> float | None:
|
||||
x, y, px, py = lonlat_to_tile(lon, lat, zoom)
|
||||
img = self._tile(zoom, x, y)
|
||||
if img is None:
|
||||
return None
|
||||
r, g, b = img.getpixel((min(int(px), TILE_SIZE - 1), min(int(py), TILE_SIZE - 1)))[:3]
|
||||
return (r * 256 + g + b / 256) - 32768
|
||||
|
||||
def fill_missing_elevations(self, points: list[TrkPt], zoom: int = DEFAULT_ZOOM) -> int:
|
||||
"""Populate ele on points lacking it. Returns count filled."""
|
||||
filled = 0
|
||||
for p in points:
|
||||
if p.ele is None:
|
||||
ele = self.elevation_at(p.lat, p.lon, zoom)
|
||||
if ele is not None:
|
||||
p.ele = round(ele, 1)
|
||||
filled += 1
|
||||
return filled
|
||||
@@ -0,0 +1,76 @@
|
||||
"""OpenAI-compatible LLM adapter for trail guide extraction.
|
||||
|
||||
Works with any OpenAI-compatible endpoint (Ollama, Groq, Gemini, OpenAI)
|
||||
via configurable base_url / model / api_key.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
from app.config import LLMConfig
|
||||
|
||||
log = logging.getLogger("trekker.llm")
|
||||
|
||||
EXTRACTION_PROMPT = """You extract structured overland trail data from text.
|
||||
Return ONLY a JSON object with these keys (use null/[] when not mentioned):
|
||||
{
|
||||
"name": string,
|
||||
"difficulty": string|null, // e.g. "easy", "moderate", "difficult", "extreme", or a rating like "7/10"
|
||||
"obstacle_types": [string], // e.g. "rock garden", "ledges", "mud", "off-camber"
|
||||
"water_crossings": [{"name": string|null, "depth_ft": number|null, "notes": string|null}],
|
||||
"gate_codes": [string],
|
||||
"vehicle_recommendations": [string],// e.g. "33-inch tires", "lockers recommended", "winch"
|
||||
"closures": [string], // seasonal or permanent closure notes
|
||||
"warnings": [string],
|
||||
"notes": string|null
|
||||
}"""
|
||||
|
||||
|
||||
class LLMService:
|
||||
def __init__(self, config: LLMConfig):
|
||||
self.config = config
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return bool(self.config.api_key or self.config.base_url)
|
||||
|
||||
def _client(self) -> OpenAI:
|
||||
return OpenAI(
|
||||
base_url=self.config.base_url or None,
|
||||
api_key=self.config.api_key or "not-needed", # local providers ignore the key
|
||||
)
|
||||
|
||||
def extract_trail_data(self, text: str) -> dict[str, Any]:
|
||||
"""Trail guide text -> structured dict. Raises on provider errors."""
|
||||
client = self._client()
|
||||
# Keep prompts inside small local-model context windows.
|
||||
snippet = text[:24000]
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": self.config.model,
|
||||
"messages": [
|
||||
{"role": "system", "content": EXTRACTION_PROMPT},
|
||||
{"role": "user", "content": snippet},
|
||||
],
|
||||
}
|
||||
try:
|
||||
resp = client.chat.completions.create(
|
||||
**kwargs, response_format={"type": "json_object"}
|
||||
)
|
||||
except Exception:
|
||||
# Some providers reject response_format; retry without it.
|
||||
resp = client.chat.completions.create(**kwargs)
|
||||
content = resp.choices[0].message.content or "{}"
|
||||
# Tolerate code-fenced JSON from smaller models.
|
||||
content = content.strip()
|
||||
if content.startswith("```"):
|
||||
content = content.strip("`")
|
||||
content = content.split("\n", 1)[1] if "\n" in content else content
|
||||
start, end = content.find("{"), content.rfind("}")
|
||||
if start != -1 and end > start:
|
||||
content = content[start : end + 1]
|
||||
return json.loads(content)
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Offline map packs: estimate, download, and serve XYZ tiles locally.
|
||||
|
||||
Tiles are stored under data/offline_maps/{pack_id}/{layer}/{z}/{x}/{y}.png.
|
||||
Downloads run in a background thread and update pack status in SQLite.
|
||||
A per-pack tile cap (settings.tiles.max_pack_tiles) keeps usage within
|
||||
tile-provider policies; see PLAN.md Phase 2 gate notes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import shutil
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import settings
|
||||
from app.models.map_pack import MapPack
|
||||
from app.storage.route_store import Store
|
||||
|
||||
log = logging.getLogger("trekker.offline_maps")
|
||||
|
||||
LAYER_URLS = {
|
||||
"base": lambda: settings.tiles.base_url,
|
||||
"terrain": lambda: settings.tiles.terrain_url,
|
||||
}
|
||||
|
||||
|
||||
def _tile_range(bbox: list[float], zoom: int) -> tuple[int, int, int, int]:
|
||||
"""(x_min, x_max, y_min, y_max) tile indices covering bbox at zoom."""
|
||||
min_lon, min_lat, max_lon, max_lat = bbox
|
||||
n = 2**zoom
|
||||
|
||||
def x_of(lon: float) -> int:
|
||||
return min(n - 1, max(0, int((lon + 180.0) / 360.0 * n)))
|
||||
|
||||
def y_of(lat: float) -> int:
|
||||
lat = max(-85.05112878, min(85.05112878, lat))
|
||||
lat_r = math.radians(lat)
|
||||
return min(n - 1, max(0, int((1.0 - math.asinh(math.tan(lat_r)) / math.pi) / 2.0 * n)))
|
||||
|
||||
return x_of(min_lon), x_of(max_lon), y_of(max_lat), y_of(min_lat)
|
||||
|
||||
|
||||
def iter_tiles(bbox: list[float], min_zoom: int, max_zoom: int):
|
||||
for z in range(min_zoom, max_zoom + 1):
|
||||
x0, x1, y0, y1 = _tile_range(bbox, z)
|
||||
for x in range(x0, x1 + 1):
|
||||
for y in range(y0, y1 + 1):
|
||||
yield z, x, y
|
||||
|
||||
|
||||
def estimate_tile_count(bbox: list[float], min_zoom: int, max_zoom: int) -> int:
|
||||
total = 0
|
||||
for z in range(min_zoom, max_zoom + 1):
|
||||
x0, x1, y0, y1 = _tile_range(bbox, z)
|
||||
total += (x1 - x0 + 1) * (y1 - y0 + 1)
|
||||
return total
|
||||
|
||||
|
||||
class OfflineMapService:
|
||||
def __init__(self, store: Store, base_dir: str | None = None):
|
||||
self.store = store
|
||||
self.base_dir = Path(base_dir or settings.offline_maps_dir)
|
||||
|
||||
def pack_dir(self, pack_id: str) -> Path:
|
||||
return self.base_dir / pack_id
|
||||
|
||||
def create_pack(self, name: str, bbox: list[float], min_zoom: int,
|
||||
max_zoom: int, layers: list[str]) -> MapPack:
|
||||
layers = [l for l in layers if l in LAYER_URLS] or ["base"]
|
||||
count = estimate_tile_count(bbox, min_zoom, max_zoom) * len(layers)
|
||||
if count > settings.tiles.max_pack_tiles:
|
||||
raise ValueError(
|
||||
f"pack would need {count} tiles, over the cap of "
|
||||
f"{settings.tiles.max_pack_tiles}; shrink the area or zoom range"
|
||||
)
|
||||
pack = MapPack(name=name, bbox=bbox, min_zoom=min_zoom, max_zoom=max_zoom,
|
||||
layers=layers, status="queued", tile_count=count)
|
||||
pack.path = str(self.pack_dir(pack.id))
|
||||
self.store.save_map_pack(pack)
|
||||
threading.Thread(target=self._download, args=(pack.id,), daemon=True).start()
|
||||
return pack
|
||||
|
||||
def _download(self, pack_id: str) -> None:
|
||||
pack = self.store.get_map_pack(pack_id)
|
||||
if pack is None:
|
||||
return
|
||||
pack.status = "downloading"
|
||||
self.store.save_map_pack(pack)
|
||||
size = 0
|
||||
done = 0
|
||||
try:
|
||||
with httpx.Client(timeout=30, headers={"User-Agent": settings.tiles.user_agent}) as client:
|
||||
for layer in pack.layers:
|
||||
url_tpl = LAYER_URLS[layer]()
|
||||
for z, x, y in iter_tiles(pack.bbox, pack.min_zoom, pack.max_zoom):
|
||||
dest = self.pack_dir(pack.id) / layer / str(z) / str(x) / f"{y}.png"
|
||||
if not dest.is_file():
|
||||
resp = client.get(url_tpl.format(z=z, x=x, y=y))
|
||||
resp.raise_for_status()
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_bytes(resp.content)
|
||||
time.sleep(0.05) # be polite to public tile servers
|
||||
size += dest.stat().st_size
|
||||
done += 1
|
||||
if done % 50 == 0:
|
||||
pack.tiles_done, pack.size_bytes = done, size
|
||||
self.store.save_map_pack(pack)
|
||||
pack.tiles_done, pack.size_bytes, pack.status = done, size, "ready"
|
||||
except Exception as e:
|
||||
log.exception("pack %s download failed", pack_id)
|
||||
pack.status, pack.error = "failed", str(e)
|
||||
self.store.save_map_pack(pack)
|
||||
|
||||
def tile_path(self, pack_id: str, layer: str, z: int, x: int, y: int) -> Path | None:
|
||||
p = self.pack_dir(pack_id) / layer / str(z) / str(x) / f"{y}.png"
|
||||
return p if p.is_file() else None
|
||||
|
||||
def find_tile(self, layer: str, z: int, x: int, y: int) -> Path | None:
|
||||
"""Search all ready packs for a tile (offline fallback serving)."""
|
||||
for pack in self.store.list_map_packs():
|
||||
if pack.status != "ready":
|
||||
continue
|
||||
p = self.tile_path(pack.id, layer, z, x, y)
|
||||
if p:
|
||||
return p
|
||||
return None
|
||||
|
||||
def delete_pack(self, pack_id: str) -> bool:
|
||||
ok = self.store.delete_map_pack(pack_id)
|
||||
shutil.rmtree(self.pack_dir(pack_id), ignore_errors=True)
|
||||
return ok
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Trail guide web scrape: URL -> page text -> draft TrailGuide.
|
||||
|
||||
Uses httpx + BeautifulSoup. If an LLM is configured the text is run through
|
||||
structured extraction; otherwise the user fills the review form from raw text.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from app.models.trail_guide import TrailGuide, WaterCrossing
|
||||
from app.services.llm import LLMService
|
||||
|
||||
log = logging.getLogger("trekker.scraper")
|
||||
|
||||
|
||||
def fetch_page_text(url: str) -> tuple[str, str]:
|
||||
"""Returns (title, readable_text)."""
|
||||
resp = httpx.get(url, timeout=30, follow_redirects=True,
|
||||
headers={"User-Agent": "Trekker/0.1 trail guide fetch"})
|
||||
resp.raise_for_status()
|
||||
soup = BeautifulSoup(resp.text, "html.parser")
|
||||
for tag in soup(["script", "style", "nav", "footer", "header", "noscript"]):
|
||||
tag.decompose()
|
||||
title = soup.title.get_text(strip=True) if soup.title else url
|
||||
text = "\n".join(
|
||||
line for line in (ln.strip() for ln in soup.get_text("\n").splitlines()) if line
|
||||
)
|
||||
return title, text
|
||||
|
||||
|
||||
def guide_from_extraction(extracted: dict, *, source: str, source_url: str | None = None,
|
||||
pdf_path: str | None = None, raw_text: str | None = None,
|
||||
fallback_name: str = "Unnamed trail guide") -> TrailGuide:
|
||||
crossings = [
|
||||
WaterCrossing(**wc) if isinstance(wc, dict) else WaterCrossing(notes=str(wc))
|
||||
for wc in extracted.get("water_crossings") or []
|
||||
]
|
||||
return TrailGuide(
|
||||
name=extracted.get("name") or fallback_name,
|
||||
source=source,
|
||||
source_url=source_url,
|
||||
pdf_path=pdf_path,
|
||||
difficulty=extracted.get("difficulty"),
|
||||
obstacle_types=[str(o) for o in extracted.get("obstacle_types") or []],
|
||||
water_crossings=crossings,
|
||||
gate_codes=[str(g) for g in extracted.get("gate_codes") or []],
|
||||
vehicle_recommendations=[str(v) for v in extracted.get("vehicle_recommendations") or []],
|
||||
closures=[str(c) for c in extracted.get("closures") or []],
|
||||
warnings=[str(w) for w in extracted.get("warnings") or []],
|
||||
notes=extracted.get("notes"),
|
||||
raw_text=raw_text,
|
||||
)
|
||||
|
||||
|
||||
def scrape_trail_guide(url: str, llm: LLMService) -> TrailGuide:
|
||||
title, text = fetch_page_text(url)
|
||||
extracted: dict = {}
|
||||
if llm.configured and text:
|
||||
try:
|
||||
extracted = llm.extract_trail_data(text)
|
||||
except Exception:
|
||||
log.exception("LLM extraction failed for %s; returning raw-text draft", url)
|
||||
return guide_from_extraction(
|
||||
extracted, source="web", source_url=url,
|
||||
raw_text=text[:50000], fallback_name=title,
|
||||
)
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Open-Meteo client and per-segment aggregation into four tiers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.models.route import TrekkerRoute
|
||||
|
||||
API_URL = "https://api.open-meteo.com/v1/forecast"
|
||||
HOURLY_VARS = "temperature_2m,precipitation_probability,precipitation,wind_speed_10m,wind_gusts_10m,weather_code"
|
||||
DAILY_VARS = "temperature_2m_max,temperature_2m_min,precipitation_probability_max,precipitation_sum,wind_speed_10m_max,weather_code"
|
||||
|
||||
# Open-Meteo WMO codes indicating thunderstorms (lightning risk)
|
||||
THUNDER_CODES = {95, 96, 99}
|
||||
WIND_ADVISORY_KMH = 40.0
|
||||
|
||||
|
||||
def fetch_forecast(lat: float, lon: float, days: int = 7) -> dict[str, Any]:
|
||||
if days not in (7, 10):
|
||||
raise ValueError("forecast window must be 7 or 10 days")
|
||||
resp = httpx.get(
|
||||
API_URL,
|
||||
params={
|
||||
"latitude": round(lat, 4),
|
||||
"longitude": round(lon, 4),
|
||||
"hourly": HOURLY_VARS,
|
||||
"daily": DAILY_VARS,
|
||||
"forecast_days": days,
|
||||
"timezone": "auto",
|
||||
"wind_speed_unit": "kmh",
|
||||
"temperature_unit": "fahrenheit",
|
||||
"precipitation_unit": "inch",
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _chunk_hours(hourly: dict[str, list], hours_per_bucket: int) -> list[dict[str, Any]]:
|
||||
times = hourly.get("time", [])
|
||||
buckets = []
|
||||
for start in range(0, len(times), hours_per_bucket):
|
||||
end = min(start + hours_per_bucket, len(times))
|
||||
sl = slice(start, end)
|
||||
|
||||
def vals(key: str) -> list[float]:
|
||||
return [v for v in hourly.get(key, [])[sl] if v is not None]
|
||||
|
||||
temps = vals("temperature_2m")
|
||||
winds = vals("wind_speed_10m")
|
||||
codes = vals("weather_code")
|
||||
precip_prob = vals("precipitation_probability")
|
||||
buckets.append({
|
||||
"time_start": times[start],
|
||||
"time_end": times[end - 1],
|
||||
"temp_min_f": min(temps) if temps else None,
|
||||
"temp_max_f": max(temps) if temps else None,
|
||||
"precip_prob_max": max(precip_prob) if precip_prob else None,
|
||||
"precip_sum_in": round(sum(vals("precipitation")), 3),
|
||||
"wind_max_kmh": max(winds) if winds else None,
|
||||
"wind_advisory": bool(winds) and max(winds) >= WIND_ADVISORY_KMH,
|
||||
"lightning_risk": any(int(c) in THUNDER_CODES for c in codes),
|
||||
})
|
||||
return buckets
|
||||
|
||||
|
||||
def build_tiers(forecast: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Hourly / 6h / 12h / daily tiers from a raw Open-Meteo response."""
|
||||
hourly = forecast.get("hourly", {})
|
||||
daily = forecast.get("daily", {})
|
||||
return {
|
||||
"hourly": _chunk_hours(hourly, 1),
|
||||
"6h": _chunk_hours(hourly, 6),
|
||||
"12h": _chunk_hours(hourly, 12),
|
||||
"daily": [
|
||||
{
|
||||
"date": daily["time"][i],
|
||||
"temp_min_f": daily["temperature_2m_min"][i],
|
||||
"temp_max_f": daily["temperature_2m_max"][i],
|
||||
"precip_prob_max": daily["precipitation_probability_max"][i],
|
||||
"precip_sum_in": daily["precipitation_sum"][i],
|
||||
"wind_max_kmh": daily["wind_speed_10m_max"][i],
|
||||
"lightning_risk": int(daily["weather_code"][i]) in THUNDER_CODES,
|
||||
}
|
||||
for i in range(len(daily.get("time", [])))
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def segment_midpoints(route: TrekkerRoute) -> list[tuple[int, float, float]]:
|
||||
out = []
|
||||
for seg in route.segments:
|
||||
mid = route.points[(seg.start_idx + seg.end_idx) // 2]
|
||||
out.append((seg.index, mid.lat, mid.lon))
|
||||
return out
|
||||
|
||||
|
||||
def fetch_route_weather(route: TrekkerRoute, days: int = 7) -> dict[str, Any]:
|
||||
"""Per-segment forecasts. Nearby midpoints share one fetch (0.05° grid)."""
|
||||
cache: dict[tuple[float, float], dict] = {}
|
||||
segments = {}
|
||||
for idx, lat, lon in segment_midpoints(route):
|
||||
key = (round(lat / 0.05) * 0.05, round(lon / 0.05) * 0.05)
|
||||
if key not in cache:
|
||||
cache[key] = build_tiers(fetch_forecast(lat, lon, days))
|
||||
segments[str(idx)] = cache[key]
|
||||
return {"days": days, "segments": segments}
|
||||
Reference in new issue
Block a user