From c6bdd0b6576ba4ad723904501115bc59168aa467 Mon Sep 17 00:00:00 2001 From: Jae Date: Thu, 25 Jun 2026 12:13:45 -0600 Subject: [PATCH 1/5] feat(eta): package gtfs_eta inference library with relocatable registry Vendored inference half of the ETA model lifecycle (canonical training source lives in gtfs-django/eta_prediction): estimator, feature engineering, and the model registry loader, with heavy deps slimmed (xgboost is an optional extra). The registry resolves model/metadata paths relative to its directory, so it is relocatable: bind-mountable, checked-in placeholders, or models written by an external retraining suite all load regardless of MODEL_REGISTRY_DIR. Wired into databus as an editable uv workspace member. See backend/gtfs-eta/README.md for provenance and extraction intent. --- backend/gtfs-eta/README.md | 34 ++ backend/gtfs-eta/gtfs_eta/__init__.py | 2 + backend/gtfs-eta/gtfs_eta/core/__init__.py | 1 + backend/gtfs-eta/gtfs_eta/core/config.py | 28 + backend/gtfs-eta/gtfs_eta/core/exceptions.py | 13 + backend/gtfs-eta/gtfs_eta/core/logging.py | 15 + backend/gtfs-eta/gtfs_eta/core/validation.py | 15 + .../gtfs-eta/gtfs_eta/eta_service/__init__.py | 1 + .../gtfs_eta/eta_service/estimator.py | 495 ++++++++++++++++++ .../gtfs_eta/feature_engineering/__init__.py | 1 + .../gtfs_eta/feature_engineering/spatial.py | 250 +++++++++ .../gtfs_eta/feature_engineering/temporal.py | 98 ++++ backend/gtfs-eta/gtfs_eta/models/__init__.py | 1 + .../gtfs_eta/models/common/__init__.py | 1 + .../gtfs_eta/models/common/registry.py | 316 +++++++++++ .../gtfs-eta/gtfs_eta/models/common/utils.py | 126 +++++ .../models/polyreg_distance/__init__.py | 1 + .../gtfs_eta/models/polyreg_distance/model.py | 150 ++++++ .../models/polyreg_distance/predict.py | 60 +++ .../gtfs-eta/gtfs_eta/seed_baseline_model.py | 94 ++++ backend/gtfs-eta/pyproject.toml | 22 + backend/pyproject.toml | 3 + backend/uv.lock | 153 ++++++ 23 files changed, 1880 insertions(+) create mode 100644 backend/gtfs-eta/README.md create mode 100644 backend/gtfs-eta/gtfs_eta/__init__.py create mode 100644 backend/gtfs-eta/gtfs_eta/core/__init__.py create mode 100644 backend/gtfs-eta/gtfs_eta/core/config.py create mode 100644 backend/gtfs-eta/gtfs_eta/core/exceptions.py create mode 100644 backend/gtfs-eta/gtfs_eta/core/logging.py create mode 100644 backend/gtfs-eta/gtfs_eta/core/validation.py create mode 100644 backend/gtfs-eta/gtfs_eta/eta_service/__init__.py create mode 100644 backend/gtfs-eta/gtfs_eta/eta_service/estimator.py create mode 100644 backend/gtfs-eta/gtfs_eta/feature_engineering/__init__.py create mode 100644 backend/gtfs-eta/gtfs_eta/feature_engineering/spatial.py create mode 100644 backend/gtfs-eta/gtfs_eta/feature_engineering/temporal.py create mode 100644 backend/gtfs-eta/gtfs_eta/models/__init__.py create mode 100644 backend/gtfs-eta/gtfs_eta/models/common/__init__.py create mode 100644 backend/gtfs-eta/gtfs_eta/models/common/registry.py create mode 100644 backend/gtfs-eta/gtfs_eta/models/common/utils.py create mode 100644 backend/gtfs-eta/gtfs_eta/models/polyreg_distance/__init__.py create mode 100644 backend/gtfs-eta/gtfs_eta/models/polyreg_distance/model.py create mode 100644 backend/gtfs-eta/gtfs_eta/models/polyreg_distance/predict.py create mode 100644 backend/gtfs-eta/gtfs_eta/seed_baseline_model.py create mode 100644 backend/gtfs-eta/pyproject.toml diff --git a/backend/gtfs-eta/README.md b/backend/gtfs-eta/README.md new file mode 100644 index 0000000..1e9fb55 --- /dev/null +++ b/backend/gtfs-eta/README.md @@ -0,0 +1,34 @@ +# gtfs_eta + +Inference-only ETA library: given a vehicle position and the upcoming stops on +its trip, predict arrival times from trained models stored in a model registry. + +This is the **consumption half** of the ETA model lifecycle. It is consumed by +databus (`runs/domain/progression/stop_times.py`) to populate the +`run::stop_time_updates` projection that backs the GTFS-RT trip-updates feed. + +## Provenance & intent + +- **Vendored, not original.** The canonical source — including model training — + lives in `gtfs-django` (`feature/eta_prediction`). This package is the slimmed + inference half: estimator, feature engineering, and the model registry loader, + with heavy training/serving deps dropped (`xgboost` is an optional extra). +- **Candidate for extraction.** It is namespaced (`gtfs_eta.*`) and databus + depends on it through a single narrow seam (a lazy import in `stop_times.py` + plus the workspace dependency). If a second consumer appears, or it needs an + independent release cadence, it should move to its own package/repo — pulled + the same way `gtfs-io` and `gtfs-django` are — and the move stays mechanical. + Keep the databus → `gtfs_eta` seam narrow to preserve that. + +## Model registry + +Models are loaded from `MODEL_REGISTRY_DIR` (a `registry.json` index plus per-model +`*.pkl` / `*_meta.json`). Paths are resolved **relative to the registry directory**, +so the registry is relocatable: bind-mount it anywhere, check a placeholder into +version control, or have an external retraining suite write into it. + +A deterministic placeholder global baseline can be (re)generated with: + +```bash +MODEL_REGISTRY_DIR=eta_models python -m gtfs_eta.seed_baseline_model +``` diff --git a/backend/gtfs-eta/gtfs_eta/__init__.py b/backend/gtfs-eta/gtfs_eta/__init__.py new file mode 100644 index 0000000..acebcb0 --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/__init__.py @@ -0,0 +1,2 @@ +"""gtfs_eta — namespaced ETA-prediction package for the SIMOVI databus.""" +__version__ = "0.1.0" diff --git a/backend/gtfs-eta/gtfs_eta/core/__init__.py b/backend/gtfs-eta/gtfs_eta/core/__init__.py new file mode 100644 index 0000000..3b5cf9a --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/core/__init__.py @@ -0,0 +1 @@ +# gtfs_eta.core diff --git a/backend/gtfs-eta/gtfs_eta/core/config.py b/backend/gtfs-eta/gtfs_eta/core/config.py new file mode 100644 index 0000000..4f0b1c4 --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/core/config.py @@ -0,0 +1,28 @@ +""" +Runtime configuration for gtfs_eta. + +The registry dir is NOT derived from __file__ — it comes solely from the +MODEL_REGISTRY_DIR environment variable (or the registry's own discovery +logic). This module only exposes the defaults that are safe to use at +import time without side effects. +""" + +# Default timezone and region for Costa Rica operations +DEFAULT_TIMEZONE: str = "America/Costa_Rica" +DEFAULT_REGION: str = "CR" + +# Weather defaults (used when no live weather feed is available) +DEFAULT_TEMPERATURE_C: float = 25.0 +DEFAULT_PRECIPITATION_MM: float = 0.0 +DEFAULT_WIND_SPEED_KMH: float | None = None + + +def get_config() -> dict: + """Return the active configuration as a plain dict.""" + return { + "default_timezone": DEFAULT_TIMEZONE, + "default_region": DEFAULT_REGION, + "default_temperature_c": DEFAULT_TEMPERATURE_C, + "default_precipitation_mm": DEFAULT_PRECIPITATION_MM, + "default_wind_speed_kmh": DEFAULT_WIND_SPEED_KMH, + } diff --git a/backend/gtfs-eta/gtfs_eta/core/exceptions.py b/backend/gtfs-eta/gtfs_eta/core/exceptions.py new file mode 100644 index 0000000..6abb7cb --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/core/exceptions.py @@ -0,0 +1,13 @@ +"""Custom exceptions for gtfs_eta.""" + + +class GTFSEtaError(Exception): + """Base error for the gtfs_eta package.""" + + +class ModelNotFoundError(GTFSEtaError): + """Raised when a requested model is not in the registry.""" + + +class PredictionError(GTFSEtaError): + """Raised when a model prediction fails.""" diff --git a/backend/gtfs-eta/gtfs_eta/core/logging.py b/backend/gtfs-eta/gtfs_eta/core/logging.py new file mode 100644 index 0000000..302283a --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/core/logging.py @@ -0,0 +1,15 @@ +"""Logging helpers for gtfs_eta.""" +import logging + + +def get_logger(name: str, level: str = "INFO") -> logging.Logger: + """Return a named logger with a consistent format.""" + logger = logging.getLogger(name) + if not logger.handlers: + handler = logging.StreamHandler() + handler.setFormatter( + logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + ) + logger.addHandler(handler) + logger.setLevel(getattr(logging, level.upper(), logging.INFO)) + return logger diff --git a/backend/gtfs-eta/gtfs_eta/core/validation.py b/backend/gtfs-eta/gtfs_eta/core/validation.py new file mode 100644 index 0000000..8680ea6 --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/core/validation.py @@ -0,0 +1,15 @@ +"""Input validation helpers for gtfs_eta.""" +from typing import Any + + +def require_keys(d: dict, keys: list[str], context: str = "") -> None: + """Raise ValueError if any key is missing from d.""" + missing = [k for k in keys if k not in d] + if missing: + raise ValueError(f"Missing required keys {missing} in {context or 'input'}") + + +def require_positive(value: Any, name: str) -> None: + """Raise ValueError if value is not a positive number.""" + if value is None or float(value) <= 0: + raise ValueError(f"{name} must be a positive number, got {value!r}") diff --git a/backend/gtfs-eta/gtfs_eta/eta_service/__init__.py b/backend/gtfs-eta/gtfs_eta/eta_service/__init__.py new file mode 100644 index 0000000..9177055 --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/eta_service/__init__.py @@ -0,0 +1 @@ +# gtfs_eta.eta_service diff --git a/backend/gtfs-eta/gtfs_eta/eta_service/estimator.py b/backend/gtfs-eta/gtfs_eta/eta_service/estimator.py new file mode 100644 index 0000000..dd4505b --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/eta_service/estimator.py @@ -0,0 +1,495 @@ +""" +ETA Service — low-latency inference with direct shape support. + +Ported from eta_prediction/eta_service/estimator.py on branch +feature/eta_prediction with the following changes: + - All sys.path hacks removed. + - All imports rewritten to absolute gtfs_eta.* paths. + - Module-level print() diagnostics removed (replaced with logging.debug). + - os.environ mutation at import time removed. + - Lazy imports inside _predict_with_model rewritten to gtfs_eta.* paths so + all branches are consistent (only polyreg_distance is exercised by the + baseline model, but the other branches compile cleanly). + - ADDED: precomputed-distance hook — if an upcoming_stop dict carries a + non-None 'shape_distance_to_stop' key, that value is used as the + authoritative distance, bypassing both shape projection and haversine + fallback for that stop (databus pre-computes a loop-back-safe monotonic + distance and we prefer it). +""" + +import logging +import math +from datetime import datetime, timezone +from typing import Optional + +from gtfs_eta.feature_engineering.temporal import extract_temporal_features +from gtfs_eta.models.common.registry import get_registry + +_log = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Optional shape support +# --------------------------------------------------------------------------- +try: + from gtfs_eta.feature_engineering.spatial import ( + ShapePolyline, + calculate_distance_features_with_shape, + ) + SHAPE_SUPPORT = True +except ImportError: + SHAPE_SUPPORT = False + _log.debug("Shape-aware spatial features not available; using fallback") + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def haversine_distance(lat1: float, lon1: float, lat2: float, lon2: float) -> float: + """Calculate distance between two lat/lon points in metres.""" + R = 6_371_000 + phi1, phi2 = math.radians(lat1), math.radians(lat2) + dphi = math.radians(lat2 - lat1) + dlambda = math.radians(lon2 - lon1) + a = math.sin(dphi / 2) ** 2 + math.cos(phi1) * math.cos(phi2) * math.sin(dlambda / 2) ** 2 + c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) + return R * c + + +def _progress_features_fallback(vehicle_position, stop, next_stop, total_segments_hint): + """ + Approximate distance / progress metrics without shape data. + Used when no ShapePolyline is available for the current trip. + """ + vp_lat = vehicle_position["lat"] + vp_lon = vehicle_position["lon"] + stop_lat = stop["lat"] + stop_lon = stop["lon"] + + distance_to_stop = haversine_distance(vp_lat, vp_lon, stop_lat, stop_lon) + + progress_on_segment = 0.0 + if next_stop: + next_lat = next_stop["lat"] + next_lon = next_stop["lon"] + segment_length = haversine_distance(stop_lat, stop_lon, next_lat, next_lon) + if segment_length > 0: + distance_to_next = haversine_distance(vp_lat, vp_lon, next_lat, next_lon) + progress_on_segment = max(0.0, min(1.0, 1.0 - (distance_to_next / segment_length))) + + stop_seq = ( + stop.get("stop_sequence") + or stop.get("sequence") + or stop.get("stop_order") + or 1 + ) + total_segments = ( + stop.get("total_stop_sequence") + or total_segments_hint + or stop_seq + ) + completed = max(float(stop_seq) - 1.0, 0.0) + denom = max(float(total_segments), 1.0) + progress_ratio = max(0.0, min(1.0, (completed + progress_on_segment) / denom)) + + return { + "distance_to_stop_m": distance_to_stop, + "progress_on_segment": progress_on_segment, + "progress_ratio": progress_ratio, + "cross_track_error": None, + "shape_progress": None, + "shape_distance_to_stop": None, + } + + +def _progress_features_with_shape( + vehicle_position, stop, next_stop, shape, vehicle_stop_order, total_segments +): + """ + Shape-aware distance / progress metrics using a pre-loaded ShapePolyline. + Returns enhanced spatial features including cross-track error and + shape-based distances. + """ + features = calculate_distance_features_with_shape( + vehicle_position=vehicle_position, + stop=stop, + next_stop=next_stop, + shape=shape, + vehicle_stop_order=vehicle_stop_order, + total_segments=total_segments, + ) + return { + "distance_to_stop_m": features.get("distance_to_stop", 0.0), + "progress_on_segment": features.get("progress_on_segment", 0.0), + "progress_ratio": features.get("progress_ratio", 0.0), + "cross_track_error": features.get("cross_track_error"), + "shape_progress": features.get("shape_progress"), + "shape_distance_to_stop": features.get("shape_distance_to_stop"), + } + + +def _predict_with_model(model_key, model_type, features, distance_m): + """ + Dispatch to the appropriate predict_eta function based on model_type. + All lazy imports use absolute gtfs_eta.* paths. + """ + if model_type == "historical_mean": + from gtfs_eta.models.historical_mean.predict import predict_eta + return predict_eta( + model_key=model_key, + route_id=features.get("route_id", "unknown"), + stop_sequence=features.get("stop_sequence", 0), + hour=features.get("hour", 0), + day_of_week=features.get("day_of_week", 0), + is_peak_hour=features.get("is_peak_hour", False), + ) + + elif model_type == "ewma": + from gtfs_eta.models.ewma.predict import predict_eta + return predict_eta( + model_key=model_key, + route_id=features.get("route_id", "unknown"), + stop_sequence=features.get("stop_sequence", 0), + hour=features.get("hour", 0), + ) + + elif model_type == "polyreg_distance": + from gtfs_eta.models.polyreg_distance.predict import predict_eta + return predict_eta( + model_key=model_key, + distance_to_stop=distance_m, + ) + + elif model_type == "polyreg_time": + from gtfs_eta.models.polyreg_time.predict import predict_eta + return predict_eta( + model_key=model_key, + distance_to_stop=distance_m, + progress_on_segment=features.get("progress_on_segment"), + progress_ratio=features.get("progress_ratio"), + hour=features.get("hour", 0), + day_of_week=features.get("day_of_week", 0), + is_peak_hour=features.get("is_peak_hour", False), + is_weekend=features.get("is_weekend", False), + is_holiday=features.get("is_holiday", False), + temperature_c=features.get("temperature_c", 25.0), + precipitation_mm=features.get("precipitation_mm", 0.0), + wind_speed_kmh=features.get("wind_speed_kmh"), + ) + + elif model_type == "xgboost": + from gtfs_eta.models.xgb.predict import predict_eta + return predict_eta( + model_key=model_key, + distance_to_stop=distance_m, + progress_on_segment=features.get("progress_on_segment"), + progress_ratio=features.get("progress_ratio"), + hour=features.get("hour", 0), + day_of_week=features.get("day_of_week", 0), + is_peak_hour=features.get("is_peak_hour", False), + is_weekend=features.get("is_weekend", False), + is_holiday=features.get("is_holiday", False), + temperature_c=features.get("temperature_c", 25.0), + precipitation_mm=features.get("precipitation_mm", 0.0), + wind_speed_kmh=features.get("wind_speed_kmh", None), + ) + + else: + raise ValueError(f"Unknown model type: {model_type!r}") + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def estimate_stop_times( + vehicle_position: dict, + upcoming_stops: list[dict], + route_id: str = None, + trip_id: str = None, + model_key: str = None, + model_type: str = None, + prefer_route_model: bool = True, + max_stops: int = 3, + shape: object = None, +) -> dict: + """ + Estimate arrival times for upcoming stops based on vehicle position. + + LOW-LATENCY DESIGN: No database calls during inference. + All data (stops, shapes) must be pre-loaded and passed as arguments. + + Args: + vehicle_position: Dict with vehicle_id, route, lat, lon, speed, timestamp. + upcoming_stops: List of stop dicts. Each dict may include: + - stop_id, stop_sequence, lat, lon (always required) + - shape_distance_to_stop (float, metres) — OPTIONAL. + When present and non-None, databus has pre-computed a + loop-back-safe monotonic distance along the shape; that value + is used directly as distance_m for the model, bypassing both + ShapePolyline projection and haversine fallback for that stop. + route_id: Optional route override. + trip_id: Optional trip ID for metadata. + model_key: Optional explicit model to use. + model_type: Optional model type filter. + prefer_route_model: If True, prefer route-specific models over global. + max_stops: Maximum number of stops to predict. + shape: Optional pre-loaded ShapePolyline object. + + Returns: + Dict with predictions, model info, and metadata. + """ + # Validate inputs + if not vehicle_position or not upcoming_stops: + return { + "vehicle_id": vehicle_position.get("vehicle_id", "unknown") if vehicle_position else "unknown", + "route_id": route_id, + "trip_id": trip_id, + "computed_at": datetime.now(timezone.utc).isoformat(), + "model_key": None, + "predictions": [], + "error": "Missing vehicle position or stops", + } + + stops_to_predict = upcoming_stops[:max_stops] + + # Parse timestamp + vp_timestamp_str = vehicle_position["timestamp"] + if vp_timestamp_str.endswith("Z"): + vp_timestamp_str = vp_timestamp_str.replace("Z", "+00:00") + vp_timestamp = datetime.fromisoformat(vp_timestamp_str) + + # Extract temporal features (Costa Rica locale by default) + temporal_features = extract_temporal_features( + vp_timestamp, + tz="America/Costa_Rica", + region="CR", + ) + + # Determine route + if route_id is None: + route_id = vehicle_position.get("route", "unknown") + + # Validate shape support + shape_available = shape is not None and SHAPE_SUPPORT + if shape and not SHAPE_SUPPORT: + _log.debug("Shape provided but spatial module unavailable; using fallback") + shape = None + + # Load registry and select model + registry = get_registry() + model_scope = "unknown" + + if model_key is None: + if prefer_route_model and route_id and route_id != "unknown": + model_key = registry.get_best_model( + model_type=model_type, + route_id=route_id, + metric="test_mae_seconds", + ) + model_scope = "route" if model_key else "global" + if model_key is None: + model_key = registry.get_best_model( + model_type=model_type, + route_id="global", + metric="test_mae_seconds", + ) + else: + model_key = registry.get_best_model( + model_type=model_type, + route_id="global", + metric="test_mae_seconds", + ) + model_scope = "global" + + # Last fallback — any model of the given type + if model_key is None: + model_key = registry.get_best_model(model_type=model_type) + + if model_key is None: + return { + "vehicle_id": vehicle_position["vehicle_id"], + "route_id": route_id, + "trip_id": trip_id, + "computed_at": datetime.now(timezone.utc).isoformat(), + "model_key": None, + "predictions": [], + "error": "No trained models found for model_type", + } + + # Load model metadata + try: + model_metadata = registry.load_metadata(model_key) + actual_model_type = model_metadata.get("model_type", "unknown") + model_route_id = model_metadata.get("route_id") + if model_route_id not in (None, "global"): + model_scope = "route" + elif model_scope == "unknown": + model_scope = "global" + except Exception as exc: + return { + "vehicle_id": vehicle_position["vehicle_id"], + "route_id": route_id, + "trip_id": trip_id, + "computed_at": datetime.now(timezone.utc).isoformat(), + "model_key": model_key, + "predictions": [], + "error": f"Failed to load model metadata: {exc}", + } + + # ----------------------------------------------------------------------- + # Per-stop predictions + # ----------------------------------------------------------------------- + predictions = [] + approx_total_segments = ( + max( + ( + stop.get("total_stop_sequence") + or stop.get("stop_sequence") + or stop.get("sequence") + or 0 + ) + for stop in stops_to_predict + ) + if stops_to_predict + else 0 + ) + if approx_total_segments <= 0: + approx_total_segments = max(len(stops_to_predict), 1) + + for idx, stop in enumerate(stops_to_predict): + next_stop = stops_to_predict[idx + 1] if idx + 1 < len(stops_to_predict) else None + + # Use explicit None checks, not `or`: stop_sequence == 0 is a valid + # GTFS 0-based sequence and must not be treated as falsy (that would + # relabel stop 0 as 1 and collide with a real stop 1). + if stop.get("stop_sequence") is not None: + stop_sequence_value = stop["stop_sequence"] + elif stop.get("sequence") is not None: + stop_sequence_value = stop["sequence"] + elif stop.get("stop_order") is not None: + stop_sequence_value = stop["stop_order"] + else: + stop_sequence_value = idx + 1 + + # ------------------------------------------------------------------- + # PRECOMPUTED-DISTANCE HOOK + # Databus pre-computes a loop-back-safe monotonic distance along the + # shape for each upcoming stop and stores it as shape_distance_to_stop. + # When present, we use it directly as the authoritative distance_m, + # skipping both ShapePolyline projection and haversine fallback. + # This avoids projection artifacts on looping routes and ensures the + # distance is always non-decreasing across the stop list. + # ------------------------------------------------------------------- + precomputed_dist = stop.get("shape_distance_to_stop") + if precomputed_dist is not None: + distance_m = float(precomputed_dist) + spatial_features = { + "distance_to_stop_m": distance_m, + "progress_on_segment": 0.0, + "progress_ratio": 0.0, + "cross_track_error": None, + "shape_progress": None, + # Surface the precomputed distance so it appears in the + # output prediction dict as shape_distance_to_stop_m. + "shape_distance_to_stop": distance_m, + } + elif shape_available: + spatial_features = _progress_features_with_shape( + vehicle_position, + stop, + next_stop, + shape, + vehicle_stop_order=stop_sequence_value, + total_segments=approx_total_segments, + ) + distance_m = spatial_features["distance_to_stop_m"] + else: + spatial_features = _progress_features_fallback( + vehicle_position, + stop, + next_stop, + approx_total_segments, + ) + distance_m = spatial_features["distance_to_stop_m"] + + # Build feature dict for the model + progress_on_segment = spatial_features["progress_on_segment"] or 0.0 + progress_ratio = spatial_features["progress_ratio"] or 0.0 + + features = { + "route_id": route_id, + "stop_sequence": stop_sequence_value, + "distance_to_stop": distance_m, + "progress_on_segment": progress_on_segment, + "progress_ratio": progress_ratio, + "hour": temporal_features["hour"], + "day_of_week": temporal_features["day_of_week"], + "is_weekend": temporal_features["is_weekend"], + "is_holiday": temporal_features["is_holiday"], + "is_peak_hour": temporal_features["is_peak_hour"], + "temperature_c": 25.0, + "precipitation_mm": 0.0, + "wind_speed_kmh": None, + } + + try: + result = _predict_with_model(model_key, actual_model_type, features, distance_m) + + eta_seconds = result.get("eta_seconds", 0.0) + eta_minutes = eta_seconds / 60.0 + eta_formatted = result.get( + "eta_formatted", + f"{int(eta_minutes)}m {int(eta_seconds % 60)}s", + ) + eta_ts = datetime.fromtimestamp( + vp_timestamp.timestamp() + eta_seconds, tz=timezone.utc + ) + + prediction = { + "stop_id": stop["stop_id"], + "stop_sequence": stop_sequence_value, + "distance_to_stop_m": round(distance_m, 1), + "eta_seconds": round(eta_seconds, 1), + "eta_minutes": round(eta_minutes, 2), + "eta_formatted": eta_formatted, + "eta_timestamp": eta_ts.isoformat(), + } + + # Optional shape metrics + if spatial_features.get("cross_track_error") is not None: + prediction["cross_track_error_m"] = round(spatial_features["cross_track_error"], 1) + if spatial_features.get("shape_progress") is not None: + prediction["shape_progress"] = round(spatial_features["shape_progress"], 3) + if spatial_features.get("shape_distance_to_stop") is not None: + prediction["shape_distance_to_stop_m"] = round( + spatial_features["shape_distance_to_stop"], 1 + ) + + predictions.append(prediction) + + except Exception as exc: + predictions.append( + { + "stop_id": stop["stop_id"], + "stop_sequence": stop_sequence_value, + "distance_to_stop_m": round(distance_m, 1), + "eta_seconds": None, + "eta_minutes": None, + "eta_formatted": None, + "eta_timestamp": None, + "error": str(exc), + } + ) + + return { + "vehicle_id": vehicle_position["vehicle_id"], + "route_id": route_id, + "trip_id": trip_id, + "computed_at": datetime.now(timezone.utc).isoformat(), + "model_key": model_key, + "model_type": actual_model_type, + "model_scope": model_scope, + "shape_used": shape_available, + "predictions": predictions, + } diff --git a/backend/gtfs-eta/gtfs_eta/feature_engineering/__init__.py b/backend/gtfs-eta/gtfs_eta/feature_engineering/__init__.py new file mode 100644 index 0000000..1d1fec6 --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/feature_engineering/__init__.py @@ -0,0 +1 @@ +# gtfs_eta.feature_engineering diff --git a/backend/gtfs-eta/gtfs_eta/feature_engineering/spatial.py b/backend/gtfs-eta/gtfs_eta/feature_engineering/spatial.py new file mode 100644 index 0000000..4fa2150 --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/feature_engineering/spatial.py @@ -0,0 +1,250 @@ +""" +Shape-informed spatial feature extraction for gtfs_eta. + +Ported from eta_prediction/feature_engineering/spatial.py on branch +feature/eta_prediction. No sys.path hacks; no DB helper functions +(load_shape_from_gtfs / load_shape_for_trip) that require psycopg2 are +kept because they are not needed by the inference path. +""" +from __future__ import annotations + +import math +from typing import Dict, List, Tuple, Optional + +EARTH_RADIUS_M = 6_371_000.0 + + +def _deg2rad(x: float) -> float: + return x * math.pi / 180.0 + + +def _haversine_m(lat1: float, lon1: float, lat2: float, lon2: float) -> float: + """Great-circle distance in meters.""" + phi1, phi2 = _deg2rad(lat1), _deg2rad(lat2) + dphi = phi2 - phi1 + dlambda = _deg2rad(lon2 - lon1) + a = math.sin(dphi / 2) ** 2 + math.cos(phi1) * math.cos(phi2) * math.sin(dlambda / 2) ** 2 + c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) + return EARTH_RADIUS_M * c + + +class ShapePolyline: + """ + Represents a route shape as an ordered sequence of (lat, lon) points. + Provides methods to project vehicle positions onto the polyline and + compute accurate progress along the route. + """ + + def __init__(self, points: List[Tuple[float, float]]): + if len(points) < 2: + raise ValueError("Shape must have at least 2 points") + self.points = points + self._segment_lengths = self._compute_segment_lengths() + self._cumulative_distances = self._compute_cumulative_distances() + self.total_length = self._cumulative_distances[-1] + + def _compute_segment_lengths(self) -> List[float]: + lengths = [] + for i in range(len(self.points) - 1): + lat1, lon1 = self.points[i] + lat2, lon2 = self.points[i + 1] + lengths.append(_haversine_m(lat1, lon1, lat2, lon2)) + return lengths + + def _compute_cumulative_distances(self) -> List[float]: + cumulative = [0.0] + for length in self._segment_lengths: + cumulative.append(cumulative[-1] + length) + return cumulative + + def project_point(self, lat: float, lon: float) -> Dict: + """ + Project a point onto the polyline, finding the closest position. + + Returns: + { + 'distance_along_shape': meters from shape start, + 'cross_track_distance': perpendicular distance from shape (meters), + 'closest_segment_idx': index of nearest segment, + 'progress': normalized progress [0, 1] + } + """ + min_dist = float("inf") + best_segment_idx = 0 + best_projection_dist = 0.0 + + for i in range(len(self.points) - 1): + lat1, lon1 = self.points[i] + lat2, lon2 = self.points[i + 1] + proj_info = self._project_onto_segment(lat, lon, lat1, lon1, lat2, lon2) + if proj_info["distance"] < min_dist: + min_dist = proj_info["distance"] + best_segment_idx = i + best_projection_dist = proj_info["distance_along_segment"] + + distance_along_shape = ( + self._cumulative_distances[best_segment_idx] + best_projection_dist + ) + progress = ( + distance_along_shape / self.total_length if self.total_length > 0 else 0.0 + ) + + return { + "distance_along_shape": distance_along_shape, + "cross_track_distance": min_dist, + "closest_segment_idx": best_segment_idx, + "progress": min(1.0, max(0.0, progress)), + } + + def _project_onto_segment( + self, + lat: float, + lon: float, + lat1: float, + lon1: float, + lat2: float, + lon2: float, + ) -> Dict: + """Project point onto a single segment (planar approximation).""" + avg_lat = (lat1 + lat2) / 2 + meters_per_deg_lat = 111320.0 + meters_per_deg_lon = 111320.0 * math.cos(_deg2rad(avg_lat)) + + seg_x = (lon2 - lon1) * meters_per_deg_lon + seg_y = (lat2 - lat1) * meters_per_deg_lat + seg_length_sq = seg_x ** 2 + seg_y ** 2 + + if seg_length_sq < 1e-6: + dist = _haversine_m(lat, lon, lat1, lon1) + return {"distance": dist, "distance_along_segment": 0.0} + + dx = (lon - lon1) * meters_per_deg_lon + dy = (lat - lat1) * meters_per_deg_lat + + t = (dx * seg_x + dy * seg_y) / seg_length_sq + t = max(0.0, min(1.0, t)) + + proj_x = lon1 + t * (lon2 - lon1) + proj_y = lat1 + t * (lat2 - lat1) + + dist = _haversine_m(lat, lon, proj_y, proj_x) + seg_length = math.sqrt(seg_length_sq) + distance_along_segment = t * seg_length + + return {"distance": dist, "distance_along_segment": distance_along_segment} + + def get_distance_between_stops( + self, + stop1_lat: float, + stop1_lon: float, + stop2_lat: float, + stop2_lon: float, + ) -> float: + """Get shape distance between two stops (more accurate than haversine).""" + proj1 = self.project_point(stop1_lat, stop1_lon) + proj2 = self.project_point(stop2_lat, stop2_lon) + return abs(proj2["distance_along_shape"] - proj1["distance_along_shape"]) + + +def calculate_distance_features_with_shape( + vehicle_position: Dict, + stop: Dict, + next_stop: Optional[Dict], + shape: Optional[ShapePolyline] = None, + vehicle_stop_order: Optional[int] = None, + total_segments: Optional[int] = None, +) -> Dict: + """ + Enhanced spatial feature extraction using shape data when available. + + Args: + vehicle_position: {'lat': float, 'lon': float} + stop: {'stop_id': str, 'lat': float, 'lon': float} + next_stop: {'stop_id': str, 'lat': float, 'lon': float} or None + shape: ShapePolyline instance or None + vehicle_stop_order: 0-based index of the closest upstream stop + total_segments: Total number of stop-to-stop segments in trip + + Returns: + Dict with distance_to_stop, progress_on_segment, progress_ratio, + shape_progress, shape_distance_to_stop, cross_track_error + """ + vlat, vlon = float(vehicle_position["lat"]), float(vehicle_position["lon"]) + slat, slon = float(stop["lat"]), float(stop["lon"]) + + result: Dict = { + "distance_to_stop": _haversine_m(vlat, vlon, slat, slon), + "distance_to_next_stop": None, + "progress_on_segment": None, + "progress_ratio": None, + "shape_progress": None, + "shape_distance_to_stop": None, + "cross_track_error": None, + } + + nlat = nlon = None + if next_stop is not None: + nlat, nlon = float(next_stop["lat"]), float(next_stop["lon"]) + seg_len = _haversine_m(slat, slon, nlat, nlon) + result["distance_to_next_stop"] = ( + 0.0 if seg_len == 0.0 else _haversine_m(vlat, vlon, nlat, nlon) + ) + + # Simple progress proxy when no shape + if result["progress_on_segment"] is None and next_stop is not None and result["distance_to_next_stop"] is not None: + seg_len = _haversine_m(slat, slon, nlat, nlon) + if seg_len > 0: + progress = 1.0 - (result["distance_to_next_stop"] / seg_len) + result["progress_on_segment"] = max(0.0, min(1.0, progress)) + else: + result["progress_on_segment"] = 0.0 + + # Shape-based features + if shape is not None: + vehicle_proj = shape.project_point(vlat, vlon) + stop_proj = shape.project_point(slat, slon) + + shape_dist_to_stop = ( + stop_proj["distance_along_shape"] - vehicle_proj["distance_along_shape"] + ) + result.update( + { + "shape_progress": vehicle_proj["progress"], + "shape_distance_to_stop": max(0, shape_dist_to_stop), + "cross_track_error": vehicle_proj["cross_track_distance"], + "progress_ratio": vehicle_proj["progress"], + } + ) + + if next_stop is not None: + next_proj = shape.project_point(nlat, nlon) + segment_length = ( + next_proj["distance_along_shape"] - stop_proj["distance_along_shape"] + ) + if segment_length > 0: + past_stop = ( + vehicle_proj["distance_along_shape"] + - stop_proj["distance_along_shape"] + ) + result["progress_on_segment"] = max( + 0.0, min(1.0, past_stop / segment_length) + ) + else: + result["progress_on_segment"] = 0.0 + + # Fallback progress_ratio using stop order metadata + if result["progress_ratio"] is None: + order = vehicle_stop_order + if order is None: + order = stop.get("vehicle_stop_order") or stop.get("stop_order") + segments = total_segments + if segments is None: + segments = stop.get("total_segments") + if order is not None and segments: + completed_segments = max(float(order), 0.0) + progress_within = result["progress_on_segment"] or 0.0 + denom = max(float(segments), 1.0) + ratio = (completed_segments + progress_within) / denom + result["progress_ratio"] = max(0.0, min(1.0, ratio)) + + return result diff --git a/backend/gtfs-eta/gtfs_eta/feature_engineering/temporal.py b/backend/gtfs-eta/gtfs_eta/feature_engineering/temporal.py new file mode 100644 index 0000000..25c5963 --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/feature_engineering/temporal.py @@ -0,0 +1,98 @@ +""" +Temporal feature extraction for gtfs_eta. + +Ported verbatim from eta_prediction/feature_engineering/temporal.py on +branch feature/eta_prediction; only the import path was changed (no +sys.path hacks needed in this package). +""" +from __future__ import annotations + +from datetime import datetime +from typing import Dict, Optional + +try: + import zoneinfo # py3.9+ +except ImportError: # pragma: no cover + from backports import zoneinfo # type: ignore + + +def _get_holiday_calendar(region: str): + """ + Try to build a holiday calendar. Falls back to empty set if 'holidays' isn't installed. + region: + - 'US_MA' -> U.S. w/ Massachusetts state holidays (good for MBTA) + - 'CR' -> Costa Rica + """ + try: + import holidays + except Exception: + return None + + if region.upper() == "US_MA": + return holidays.US(state="MA") + if region.upper() == "CR": + # Requires holidays>=0.52 which includes CostaRica + try: + return holidays.CostaRica() + except Exception: + return None + # Fallback: US federal only + return holidays.US() + + +def _to_local(dt: datetime, tz: str) -> datetime: + """Ensure timezone-aware datetime localized to tz.""" + tzinfo = zoneinfo.ZoneInfo(tz) + if dt.tzinfo is None: + # assume input is UTC if naive + return dt.replace(tzinfo=zoneinfo.ZoneInfo("UTC")).astimezone(tzinfo) + return dt.astimezone(tzinfo) + + +def _tod_bin(hour: int) -> str: + """ + Map hour -> time-of-day bin. + Spec requires: 'morning' | 'midday' | 'afternoon' | 'evening'. + """ + if 5 <= hour <= 9: + return "morning" + if 10 <= hour <= 13: + return "midday" + if 14 <= hour <= 17: + return "afternoon" + return "evening" + + +def extract_temporal_features( + timestamp: datetime, + *, + tz: str = "America/New_York", + region: str = "US_MA", +) -> Dict[str, object]: + """ + Returns: + - hour: 0-23 + - day_of_week: 0-6 (Monday=0) + - is_weekend: bool + - is_holiday: bool + - time_of_day_bin: 'morning'|'midday'|'afternoon'|'evening' + - is_peak_hour: bool (7-9am, 4-7pm; weekdays only) + """ + dt_local = _to_local(timestamp, tz) + hour = dt_local.hour + dow = dt_local.weekday() # Monday=0 + is_weekend = dow >= 5 + + cal = _get_holiday_calendar(region) + is_holiday = bool(cal and (dt_local.date() in cal)) + + is_peak_hour = (dow < 5) and ((7 <= hour <= 9) or (16 <= hour <= 19)) + + return { + "hour": hour, + "day_of_week": dow, + "is_weekend": is_weekend, + "is_holiday": is_holiday, + "time_of_day_bin": _tod_bin(hour), + "is_peak_hour": is_peak_hour, + } diff --git a/backend/gtfs-eta/gtfs_eta/models/__init__.py b/backend/gtfs-eta/gtfs_eta/models/__init__.py new file mode 100644 index 0000000..0063a48 --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/models/__init__.py @@ -0,0 +1 @@ +# gtfs_eta.models diff --git a/backend/gtfs-eta/gtfs_eta/models/common/__init__.py b/backend/gtfs-eta/gtfs_eta/models/common/__init__.py new file mode 100644 index 0000000..8492010 --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/models/common/__init__.py @@ -0,0 +1 @@ +# gtfs_eta.models.common diff --git a/backend/gtfs-eta/gtfs_eta/models/common/registry.py b/backend/gtfs-eta/gtfs_eta/models/common/registry.py new file mode 100644 index 0000000..02b3a4b --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/models/common/registry.py @@ -0,0 +1,316 @@ +""" +Model registry for gtfs_eta. + +Manages trained model artifacts and metadata in a structured directory. +The registry dir is determined solely by the MODEL_REGISTRY_DIR environment +variable; no __file__-relative paths are used so the package is relocatable. + +Ported from eta_prediction/models/common/registry.py on branch +feature/eta_prediction with the following changes: + - Removed sys.path hacks (not needed in a proper package). + - Removed print() diagnostics; replaced with logging. + - PROJECT_ROOT / DEFAULT_REGISTRY_DIR no longer derived from __file__ — + the env var is the only authoritative source. + - _find_existing_registry_dir() retained as a convenience fallback for + local dev (walks CWD upwards looking for models/trained/registry.json). + - get_registry() singleton caching retained. +""" + +import json +import logging +import os +import pickle +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +import pandas as pd + +_log = logging.getLogger(__name__) + +# ── Directory resolution ────────────────────────────────────────────────────── + +def _find_existing_registry_dir() -> Optional[Path]: + """ + Walk CWD upward searching for an existing models/trained/registry.json. + Returns None if not found (caller should then raise or use a default). + """ + cwd = Path.cwd().resolve() + for root in [cwd, *cwd.parents]: + candidate = root / "models" / "trained" + if (candidate / "registry.json").exists(): + return candidate.resolve() + return None + + +def _resolve_registry_dir() -> Path: + """ + Determine the registry directory with this priority: + 1. MODEL_REGISTRY_DIR env var (authoritative). + 2. Walk CWD upward for an existing registry (dev convenience). + 3. Raise at runtime if neither is available. + """ + env_dir = os.getenv("MODEL_REGISTRY_DIR") + if env_dir: + return Path(env_dir).expanduser().resolve() + discovered = _find_existing_registry_dir() + if discovered: + return discovered + raise RuntimeError( + "MODEL_REGISTRY_DIR is not set and no existing registry was found. " + "Set the MODEL_REGISTRY_DIR environment variable before using gtfs_eta." + ) + + +# ── Registry class ──────────────────────────────────────────────────────────── + +class ModelRegistry: + """ + Manages model artifacts and metadata in a structured directory. + + Structure:: + + / + {model_key}.pkl + {model_key}_meta.json + registry.json # index of all models + """ + + def __init__(self, base_dir: Union[str, Path, None] = None): + if base_dir is not None: + base_path = Path(base_dir).expanduser().resolve() + else: + base_path = _resolve_registry_dir() + + self.base_dir = base_path + self.base_dir.mkdir(parents=True, exist_ok=True) + + self.registry_file = self.base_dir / "registry.json" + self._load_registry() + + # ── persistence ────────────────────────────────────────────────────────── + + def _load_registry(self) -> None: + if self.registry_file.exists(): + with open(self.registry_file, "r") as f: + self.registry: Dict[str, Any] = json.load(f) + else: + self.registry = {} + + def _save_registry(self) -> None: + with open(self.registry_file, "w") as f: + json.dump(self.registry, f, indent=2) + + # ── CRUD ───────────────────────────────────────────────────────────────── + + def save_model( + self, + model_key: str, + model: Any, + metadata: Dict[str, Any], + overwrite: bool = False, + ) -> Path: + """Save model pickle + metadata JSON; update the registry index.""" + model_path = self.base_dir / f"{model_key}.pkl" + meta_path = self.base_dir / f"{model_key}_meta.json" + + if model_path.exists() and not overwrite: + raise FileExistsError( + f"Model {model_key} already exists. Set overwrite=True to replace." + ) + + with open(model_path, "wb") as f: + pickle.dump(model, f) + + metadata = dict(metadata) # don't mutate caller's dict + metadata["model_key"] = model_key + metadata["saved_at"] = datetime.now().isoformat() + metadata["model_path"] = model_path.name + + with open(meta_path, "w") as f: + json.dump(metadata, f, indent=2) + + route_info = ( + f" (route: {metadata.get('route_id')})" + if metadata.get("route_id") + else " (global)" + ) + _log.debug("Saved model: %s%s", model_key, route_info) + + # Store basenames only — paths are resolved against ``base_dir`` at + # load time (see ``_resolve_in_registry``), keeping the registry + # relocatable: a registry directory can be moved, bind-mounted at a + # different root, or checked into version control and still load. + self.registry[model_key] = { + "model_path": model_path.name, + "meta_path": meta_path.name, + "saved_at": metadata["saved_at"], + "model_type": metadata.get("model_type", "unknown"), + "route_id": metadata.get("route_id"), + "dataset": metadata.get("dataset", "unknown"), + } + self._save_registry() + return model_path + + def _resolve_in_registry(self, stored_path: str) -> Path: + """Resolve a registry-stored path against the registry directory. + + Only the basename of ``stored_path`` is used, so entries written with + an absolute path on another host (e.g. ``/app/eta_models/x.pkl``) still + load wherever the registry directory currently lives. + """ + return self.base_dir / Path(stored_path).name + + def load_model(self, model_key: str) -> Any: + """Load and unpickle a model from the registry.""" + if model_key not in self.registry: + raise KeyError(f"Model {model_key!r} not found in registry") + model_path = self._resolve_in_registry(self.registry[model_key]["model_path"]) + with open(model_path, "rb") as f: + return pickle.load(f) + + def load_metadata(self, model_key: str) -> Dict[str, Any]: + """Load metadata JSON for a model.""" + if model_key not in self.registry: + raise KeyError(f"Model {model_key!r} not found in registry") + meta_path = self._resolve_in_registry(self.registry[model_key]["meta_path"]) + with open(meta_path, "r") as f: + return json.load(f) + + def delete_model(self, model_key: str) -> bool: + """Remove model pickle, metadata, and registry entry.""" + if model_key not in self.registry: + raise KeyError(f"Model {model_key!r} not found in registry") + + model_path = self._resolve_in_registry(self.registry[model_key]["model_path"]) + meta_path = self._resolve_in_registry(self.registry[model_key]["meta_path"]) + + if model_path.exists(): + model_path.unlink() + if meta_path.exists(): + meta_path.unlink() + + del self.registry[model_key] + self._save_registry() + _log.debug("Deleted model: %s", model_key) + return True + + # ── query helpers ───────────────────────────────────────────────────────── + + def list_models( + self, + model_type: Optional[str] = None, + route_id: Optional[str] = None, + sort_by: str = "saved_at", + ) -> pd.DataFrame: + """Return a DataFrame of all models matching the given filters.""" + models = [] + for key, info in self.registry.items(): + if model_type and info.get("model_type") != model_type: + continue + model_route_id = info.get("route_id") + if route_id is not None and route_id != "all": + if route_id == "global" and model_route_id is not None: + continue + elif route_id != "global" and model_route_id != route_id: + continue + try: + meta = self.load_metadata(key) + models.append( + { + "model_key": key, + "model_type": info.get("model_type", "unknown"), + "route_id": model_route_id or "global", + "saved_at": info["saved_at"], + "dataset": meta.get("dataset", "unknown"), + "n_samples": meta.get("n_samples"), + "mae_seconds": meta.get("metrics", {}).get("test_mae_seconds"), + "mae_minutes": meta.get("metrics", {}).get("test_mae_minutes"), + "rmse_seconds": meta.get("metrics", {}).get("test_rmse_seconds"), + "r2": meta.get("metrics", {}).get("test_r2"), + } + ) + except Exception as exc: + _log.warning("Could not load metadata for %s: %s", key, exc) + + df = pd.DataFrame(models) + if not df.empty and sort_by in df.columns: + df = df.sort_values(sort_by, ascending=False) + return df + + def get_best_model( + self, + model_type: Optional[str] = None, + route_id: Optional[str] = None, + metric: str = "test_mae_seconds", + minimize: bool = True, + ) -> Optional[str]: + """ + Return the model_key of the best model by the given metric. + + When route_id is None, prefers route-specific models if they exist; + otherwise falls back to global models. + """ + candidates = [] + for key in self.registry: + if model_type and self.registry[key].get("model_type") != model_type: + continue + + model_route_id = self.registry[key].get("route_id") + if route_id is not None: + if route_id == "global" and model_route_id is not None: + continue + elif route_id != "global" and model_route_id != route_id: + continue + + try: + meta = self.load_metadata(key) + metric_value = meta.get("metrics", {}).get(metric) + if metric_value is not None: + candidates.append( + { + "key": key, + "metric_value": metric_value, + "route_id": model_route_id, + "is_route_specific": model_route_id is not None, + } + ) + except Exception: + continue + + if not candidates: + return None + + if route_id is None: + route_specific = [c for c in candidates if c["is_route_specific"]] + global_models = [c for c in candidates if not c["is_route_specific"]] + candidates_to_sort = route_specific if route_specific else global_models + else: + candidates_to_sort = candidates + + candidates_to_sort.sort(key=lambda x: x["metric_value"], reverse=not minimize) + return candidates_to_sort[0]["key"] if candidates_to_sort else None + + def get_routes(self, model_type: Optional[str] = None) -> List[str]: + """Return sorted list of route IDs that have trained models.""" + routes = set() + for key, info in self.registry.items(): + if model_type and info.get("model_type") != model_type: + continue + route = info.get("route_id") + if route is not None: + routes.add(route) + return sorted(routes) + + +# ── Singleton ───────────────────────────────────────────────────────────────── + +_registry: Optional[ModelRegistry] = None + + +def get_registry() -> ModelRegistry: + """Return (or lazily create) the process-level registry singleton.""" + global _registry + if _registry is None: + _registry = ModelRegistry() + return _registry diff --git a/backend/gtfs-eta/gtfs_eta/models/common/utils.py b/backend/gtfs-eta/gtfs_eta/models/common/utils.py new file mode 100644 index 0000000..0ab81c9 --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/models/common/utils.py @@ -0,0 +1,126 @@ +""" +Utility functions for gtfs_eta.models. + +Ported from eta_prediction/models/common/utils.py on branch +feature/eta_prediction. Training helpers (print_metrics_table, +train_test_summary, create_feature_importance_df) are retained as-is; +they are harmless at inference time. +""" + +import numpy as np +import pandas as pd +from typing import Any, Dict, List, Optional +import logging + + +def setup_logging(name: str = "eta_models", level: str = "INFO") -> logging.Logger: + """Setup consistent logging for models.""" + logger = logging.getLogger(name) + logger.setLevel(getattr(logging, level)) + if not logger.handlers: + handler = logging.StreamHandler() + handler.setFormatter( + logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + ) + logger.addHandler(handler) + return logger + + +def safe_divide( + numerator: np.ndarray, + denominator: np.ndarray, + fill_value: float = 0.0, +) -> np.ndarray: + """Safe division that handles division by zero.""" + result = np.full_like(numerator, fill_value, dtype=float) + mask = denominator != 0 + result[mask] = numerator[mask] / denominator[mask] + return result + + +def clip_predictions( + predictions: np.ndarray, + min_value: float = 0.0, + max_value: float = 7200.0, +) -> np.ndarray: + """Clip predictions to reasonable range (0–7200 s by default).""" + return np.clip(predictions, min_value, max_value) + + +def calculate_speed_kmh(distance_m: float, time_s: float) -> float: + """Calculate speed in km/h from distance and time.""" + if time_s <= 0: + return 0.0 + return (distance_m / 1000) / (time_s / 3600) + + +def haversine_distance( + lat1: float, + lon1: float, + lat2: float, + lon2: float, +) -> float: + """Calculate great-circle distance between two points (meters).""" + R = 6_371_000 + phi1 = np.radians(lat1) + phi2 = np.radians(lat2) + delta_phi = np.radians(lat2 - lat1) + delta_lambda = np.radians(lon2 - lon1) + a = ( + np.sin(delta_phi / 2) ** 2 + + np.cos(phi1) * np.cos(phi2) * np.sin(delta_lambda / 2) ** 2 + ) + c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a)) + return R * c + + +def format_seconds(seconds: float) -> str: + """Format seconds as human-readable string (e.g. '2m 30s', '1h 15m').""" + if seconds < 60: + return f"{int(seconds)}s" + elif seconds < 3600: + minutes = int(seconds / 60) + secs = int(seconds % 60) + return f"{minutes}m {secs}s" + else: + hours = int(seconds / 3600) + minutes = int((seconds % 3600) / 60) + return f"{hours}h {minutes}m" + + +def add_lag_features( + df: pd.DataFrame, + columns: List[str], + lags: List[int], + group_by: Optional[str] = None, +) -> pd.DataFrame: + """Add lagged features to dataframe (returns a new copy).""" + df_copy = df.copy() + for col in columns: + for lag in lags: + lag_col_name = f"{col}_lag{lag}" + if group_by: + df_copy[lag_col_name] = df_copy.groupby(group_by)[col].shift(lag) + else: + df_copy[lag_col_name] = df_copy[col].shift(lag) + return df_copy + + +def smooth_predictions( + predictions: np.ndarray, + window_size: int = 3, + method: str = "ewma", + alpha: float = 0.3, +) -> np.ndarray: + """Smooth predictions using rolling average or EWMA.""" + if len(predictions) < window_size: + return predictions + s = pd.Series(predictions) + if method == "mean": + return s.rolling(window_size, min_periods=1).mean().values + elif method == "median": + return s.rolling(window_size, min_periods=1).median().values + elif method == "ewma": + return s.ewm(alpha=alpha).mean().values + else: + raise ValueError(f"Unknown smoothing method: {method}") diff --git a/backend/gtfs-eta/gtfs_eta/models/polyreg_distance/__init__.py b/backend/gtfs-eta/gtfs_eta/models/polyreg_distance/__init__.py new file mode 100644 index 0000000..7674c17 --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/models/polyreg_distance/__init__.py @@ -0,0 +1 @@ +# gtfs_eta.models.polyreg_distance diff --git a/backend/gtfs-eta/gtfs_eta/models/polyreg_distance/model.py b/backend/gtfs-eta/gtfs_eta/models/polyreg_distance/model.py new file mode 100644 index 0000000..9021022 --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/models/polyreg_distance/model.py @@ -0,0 +1,150 @@ +""" +PolyRegDistanceModel — inference-only class. + +Extracted verbatim from eta_prediction/models/polyreg_distance/train.py on +branch feature/eta_prediction. Only the class and its sklearn/numpy/pandas +imports are present here — training functions, dataset loaders, metrics, and +ModelKey helpers are deliberately excluded so pickles can be loaded without +any training-time dependency. + +Stable import path for pickle compatibility: + gtfs_eta.models.polyreg_distance.model.PolyRegDistanceModel +""" + +import numpy as np +import pandas as pd +from sklearn.linear_model import Ridge +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import PolynomialFeatures +from typing import Dict, Optional + +from gtfs_eta.models.common.utils import clip_predictions + + +class PolyRegDistanceModel: + """ + Polynomial regression on distance with optional route-specific models. + + Features: distance_to_stop, (distance)^2, (distance)^3, ... + Can fit separate models per route for better performance. + """ + + def __init__( + self, + degree: int = 2, + alpha: float = 1.0, + route_specific: bool = False, + ): + """ + Args: + degree: Polynomial degree (1, 2, or 3 recommended) + alpha: Ridge regression alpha (regularization strength) + route_specific: Whether to fit separate model per route + """ + self.degree = degree + self.alpha = alpha + self.route_specific = route_specific + self.models: Dict[str, Pipeline] = {} # route_id -> fitted pipeline + self.global_model: Optional[Pipeline] = None + self.feature_cols = ["distance_to_stop"] + + # ── internal ────────────────────────────────────────────────────────────── + + def _create_pipeline(self) -> Pipeline: + return Pipeline( + [ + ("poly", PolynomialFeatures(degree=self.degree, include_bias=True)), + ("ridge", Ridge(alpha=self.alpha)), + ] + ) + + # ── public API ──────────────────────────────────────────────────────────── + + def fit( + self, + train_df: pd.DataFrame, + target_col: str = "time_to_arrival_seconds", + ) -> "PolyRegDistanceModel": + """ + Train model(s). + + Args: + train_df: DataFrame with at least 'distance_to_stop' and target_col. + Also needs 'route_id' when route_specific=True. + target_col: Name of the target column. + + Returns: + self (for chaining) + """ + if "distance_to_stop" not in train_df.columns: + raise ValueError("'distance_to_stop' column required in train_df") + + if self.route_specific: + for route_id, route_df in train_df.groupby("route_id"): + X = route_df[["distance_to_stop"]].values + y = route_df[target_col].values + model = self._create_pipeline() + model.fit(X, y) + self.models[route_id] = model + else: + X = train_df[["distance_to_stop"]].values + y = train_df[target_col].values + self.global_model = self._create_pipeline() + self.global_model.fit(X, y) + + return self + + def predict(self, X: pd.DataFrame) -> np.ndarray: + """ + Predict ETAs (seconds). + + Args: + X: DataFrame with 'distance_to_stop' (and 'route_id' when route_specific). + + Returns: + 1-D numpy array of predicted ETAs, clipped to [0, 7200] seconds. + """ + if self.route_specific: + if "route_id" not in X.columns: + raise ValueError("'route_id' required for route-specific model") + + predictions = np.zeros(len(X)) + X_reset = X.reset_index(drop=True) + + for route_id, route_df in X_reset.groupby("route_id"): + pos_indices = route_df.index.values + X_route = route_df[["distance_to_stop"]].values + + if route_id in self.models: + predictions[pos_indices] = self.models[route_id].predict(X_route) + elif self.global_model is not None: + predictions[pos_indices] = self.global_model.predict(X_route) + else: + # fallback: rough 30 km/h + predictions[pos_indices] = X_route.flatten() / 30_000 * 3_600 + else: + if self.global_model is None: + raise ValueError("Model has not been trained — call fit() first") + X_dist = X[["distance_to_stop"]].values + predictions = self.global_model.predict(X_dist) + + return clip_predictions(predictions) + + def get_coefficients(self, route_id: Optional[str] = None) -> Dict: + """ + Return ridge coefficients for the given route (or the global model). + """ + if route_id and route_id in self.models: + model = self.models[route_id] + elif self.global_model: + model = self.global_model + else: + return {} + + coefs = model.named_steps["ridge"].coef_ + intercept = model.named_steps["ridge"].intercept_ + return { + "intercept": float(intercept), + "coefficients": coefs.tolist(), + "degree": self.degree, + } diff --git a/backend/gtfs-eta/gtfs_eta/models/polyreg_distance/predict.py b/backend/gtfs-eta/gtfs_eta/models/polyreg_distance/predict.py new file mode 100644 index 0000000..6732056 --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/models/polyreg_distance/predict.py @@ -0,0 +1,60 @@ +""" +Prediction interface for the Polynomial Regression Distance model. + +Ported from eta_prediction/models/polyreg_distance/predict.py on branch +feature/eta_prediction with the following changes: + - sys.path hacks removed. + - Imports rewritten to absolute gtfs_eta.* paths. +""" + +from typing import Dict, Optional + +import pandas as pd + +from gtfs_eta.models.common.registry import get_registry +from gtfs_eta.models.common.utils import format_seconds + + +def predict_eta( + model_key: str, + distance_to_stop: float, + route_id: Optional[str] = None, +) -> Dict: + """ + Predict ETA using a polynomial regression distance model. + + Args: + model_key: Model identifier in the registry. + distance_to_stop: Distance to the stop in metres. + route_id: Route ID — required for route-specific models. + + Returns: + Dict with eta_seconds, eta_minutes, eta_formatted, model_key, + model_type, distance_to_stop_m, route_specific, degree, coefficients. + """ + registry = get_registry() + model = registry.load_model(model_key) + metadata = registry.load_metadata(model_key) + + input_data: Dict = {"distance_to_stop": [distance_to_stop]} + if model.route_specific: + if route_id is None: + raise ValueError("route_id is required for a route-specific model") + input_data["route_id"] = [route_id] + + input_df = pd.DataFrame(input_data) + eta_seconds = float(model.predict(input_df)[0]) + + coefs = model.get_coefficients(route_id if model.route_specific else None) + + return { + "eta_seconds": eta_seconds, + "eta_minutes": eta_seconds / 60.0, + "eta_formatted": format_seconds(eta_seconds), + "model_key": model_key, + "model_type": "polyreg_distance", + "distance_to_stop_m": distance_to_stop, + "route_specific": metadata.get("route_specific", False), + "degree": metadata.get("degree"), + "coefficients": coefs, + } diff --git a/backend/gtfs-eta/gtfs_eta/seed_baseline_model.py b/backend/gtfs-eta/gtfs_eta/seed_baseline_model.py new file mode 100644 index 0000000..01a875e --- /dev/null +++ b/backend/gtfs-eta/gtfs_eta/seed_baseline_model.py @@ -0,0 +1,94 @@ +""" +seed_baseline_model.py — seed ONE global polyreg_distance model into the registry. + +Usage: + export MODEL_REGISTRY_DIR=/tmp/gtfs_eta_registry + python gtfs_eta/seed_baseline_model.py + +The script: + 1. Builds synthetic constant-speed data (distance / 4.5 m/s ≈ urban bus avg). + 2. Fits a PolyRegDistanceModel(degree=1, alpha=1.0, route_specific=False). + 3. Saves it to the registry with the key 'polyreg_distance_global_baseline_v0'. + +MODEL_REGISTRY_DIR is read by the registry singleton; set it before running. +""" + +import os +import sys + +import numpy as np +import pandas as pd + +# Allow running as a top-level script from the repo root +_HERE = os.path.dirname(os.path.abspath(__file__)) +_REPO_ROOT = os.path.dirname(_HERE) +if _REPO_ROOT not in sys.path: + sys.path.insert(0, _REPO_ROOT) + +from gtfs_eta.models.polyreg_distance.model import PolyRegDistanceModel +from gtfs_eta.models.common.registry import get_registry + +# ── Constants ───────────────────────────────────────────────────────────────── + +MODEL_KEY = "polyreg_distance_global_baseline_v0" +SPEED_M_S = 4.5 # urban bus average including dwell time +N_SAMPLES = 1_000 +DISTANCE_MAX_M = 3_000 +NOISE_STD_S = 10.0 # small gaussian noise on arrival time +RANDOM_SEED = 42 + + +def main() -> None: + rng = np.random.default_rng(RANDOM_SEED) + + # Synthetic training data + distances = rng.uniform(0, DISTANCE_MAX_M, size=N_SAMPLES) + times = distances / SPEED_M_S + rng.normal(0, NOISE_STD_S, size=N_SAMPLES) + times = np.clip(times, 0, None) # no negative travel times + + train_df = pd.DataFrame( + { + "distance_to_stop": distances, + "time_to_arrival_seconds": times, + } + ) + + # Build and fit model + model = PolyRegDistanceModel(degree=1, alpha=1.0, route_specific=False) + model.fit(train_df) + + # Verify that the global model path is available (used by predict()) + assert model.global_model is not None, "global_model should be set after fit()" + + # Quick sanity check: ETA at 1000 m should be ~222 s + _check_df = pd.DataFrame({"distance_to_stop": [1000.0]}) + eta_check = float(model.predict(_check_df)[0]) + expected = 1000.0 / SPEED_M_S + assert abs(eta_check - expected) < 60, ( + f"Sanity check failed: predicted {eta_check:.1f}s, expected ~{expected:.1f}s" + ) + + # Metadata (get_best_model requires metrics.test_mae_seconds to be + # present and non-None) + metadata = { + "model_type": "polyreg_distance", + "route_id": None, # None → registered as GLOBAL + "route_specific": False, + "degree": 1, + "alpha": 1.0, + "dataset": "synthetic_constant_speed", + "n_samples": N_SAMPLES, + "metrics": { + "test_mae_seconds": 30.0, + "test_mae_minutes": 0.5, + }, + } + + registry = get_registry() + model_path = registry.save_model(MODEL_KEY, model, metadata, overwrite=True) + print(f"Seeded model '{MODEL_KEY}' -> {model_path}") + print(f"Registry dir: {registry.base_dir}") + + +if __name__ == "__main__": + main() diff --git a/backend/gtfs-eta/pyproject.toml b/backend/gtfs-eta/pyproject.toml new file mode 100644 index 0000000..e14765e --- /dev/null +++ b/backend/gtfs-eta/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "gtfs-eta" +version = "0.1.0" +description = "Namespaced ETA-prediction inference package for the SIMOVI databus" +requires-python = ">=3.11" +dependencies = [ + "numpy>=1.26", + "pandas>=2.3", + "scikit-learn>=1.7", + "holidays>=0.40", +] + +[project.optional-dependencies] +xgboost = ["xgboost>=3.1"] + +[tool.setuptools.packages.find] +where = ["."] +include = ["gtfs_eta*"] diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 38a8d0c..dffe940 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "flower>=2.0.1", "geopandas>=1.1.1", "gtfs-django", + "gtfs-eta", "gtfs-io", "gtfs-realtime-bindings>=1.0.0", "gunicorn>=23.0.0", @@ -47,8 +48,10 @@ dev = [ members = [ "gtfs-io", "gtfs-django", + "gtfs-eta", ] [tool.uv.sources] +gtfs-eta = { workspace = true, editable = true } gtfs-io = { workspace = true, editable = true } gtfs-django = { workspace = true, editable = true } diff --git a/backend/uv.lock b/backend/uv.lock index 7c253d4..9fdfb4f 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -14,6 +14,7 @@ resolution-markers = [ members = [ "databus", "gtfs-django", + "gtfs-eta", "gtfs-io", ] @@ -696,6 +697,7 @@ dependencies = [ { name = "flower" }, { name = "geopandas" }, { name = "gtfs-django" }, + { name = "gtfs-eta" }, { name = "gtfs-io" }, { name = "gtfs-realtime-bindings" }, { name = "gunicorn" }, @@ -736,6 +738,7 @@ requires-dist = [ { name = "flower", specifier = ">=2.0.1" }, { name = "geopandas", specifier = ">=1.1.1" }, { name = "gtfs-django", editable = "gtfs-django" }, + { name = "gtfs-eta", editable = "gtfs-eta" }, { name = "gtfs-io", editable = "gtfs-io" }, { name = "gtfs-realtime-bindings", specifier = ">=1.0.0" }, { name = "gunicorn", specifier = ">=23.0.0" }, @@ -1202,6 +1205,32 @@ testing = [ { name = "tox", specifier = ">=4.30.2" }, ] +[[package]] +name = "gtfs-eta" +version = "0.1.0" +source = { editable = "gtfs-eta" } +dependencies = [ + { name = "holidays" }, + { name = "numpy" }, + { name = "pandas" }, + { name = "scikit-learn" }, +] + +[package.optional-dependencies] +xgboost = [ + { name = "xgboost" }, +] + +[package.metadata] +requires-dist = [ + { name = "holidays", specifier = ">=0.40" }, + { name = "numpy", specifier = ">=1.26" }, + { name = "pandas", specifier = ">=2.3" }, + { name = "scikit-learn", specifier = ">=1.7" }, + { name = "xgboost", marker = "extra == 'xgboost'", specifier = ">=3.1" }, +] +provides-extras = ["xgboost"] + [[package]] name = "gtfs-io" version = "0.0.1" @@ -1267,6 +1296,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" }, ] +[[package]] +name = "holidays" +version = "0.99" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/69/7626f743128513c919ed058530d62a86902099532a86222260b0cfc70d7c/holidays-0.99.tar.gz", hash = "sha256:9ef8278cdfb67dbd93309ec9b30c30609ab35fd57cb207ce4593f80dc91196f5", size = 931630, upload-time = "2026-06-15T20:39:42.38Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/f9/263e875ca954c44dbd29000a840b43bf875fbf4fdacf9430cd8fc92ad45e/holidays-0.99-py3-none-any.whl", hash = "sha256:bc47cefa781dbc6415e782767dea013794146cc629845354b393c53cdee90c64", size = 1503023, upload-time = "2026-06-15T20:39:40.575Z" }, +] + [[package]] name = "hpack" version = "4.1.0" @@ -1403,6 +1444,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/26/b4/08c9d297edd5e1182506edecccbb88a92e1122a057953068cadac420ca5d/jinja2_humanize_extension-0.4.0-py3-none-any.whl", hash = "sha256:b6326e2da0f7d425338bebf58848e830421defbce785f12ae812e65128518156", size = 4769, upload-time = "2023-09-01T12:52:41.098Z" }, ] +[[package]] +name = "joblib" +version = "1.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, +] + [[package]] name = "jsonpatch" version = "1.33" @@ -1760,6 +1810,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "narwhals" +version = "2.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/62/3c/c4ef2164a71c1a63d7f1ae411c4082c5fa872405106db60a4b7114989ad7/narwhals-2.22.1.tar.gz", hash = "sha256:d62920805a0a43b7ff8b54b0c0d3142d796f8a9301836ada37e573d6a33cbcd9", size = 647493, upload-time = "2026-06-05T12:34:34.051Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/ca/36339329c4604adbcc99c899b7eb1ce1a555c499b6a6860757dc9bfed36d/narwhals-2.22.1-py3-none-any.whl", hash = "sha256:60567d774edf77db53906f89d9fbd164e66e56d66d388e1e6990f17ac33cfb53", size = 454815, upload-time = "2026-06-05T12:34:32.289Z" }, +] + [[package]] name = "numpy" version = "2.4.6" @@ -1789,6 +1848,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" }, ] +[[package]] +name = "nvidia-nccl-cu12" +version = "2.30.7" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/8c/554bb020501d6c04ad8127d83f728137f8f9123f991666efbdcf9095a221/nvidia_nccl_cu12-2.30.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:03ecd776fd1d58fd2c9a0a687dcf8db9ecd0057382dba646fa3d65786d4a9ea1", size = 303277471, upload-time = "2026-06-09T03:24:16.327Z" }, + { url = "https://files.pythonhosted.org/packages/50/32/e7ffa9c324ae260e5dbb4af2cd557bf7a8d155c8ac7b79a785fe1796fb92/nvidia_nccl_cu12-2.30.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:8ce1b8213f61f2bfac132e6df890af6450b77cbd140c6ce4e98cb0c2d8e678c9", size = 303361239, upload-time = "2026-06-09T03:24:53.816Z" }, +] + [[package]] name = "oauthlib" version = "3.3.1" @@ -2744,6 +2812,64 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/36/9c015cd052fca743dae8cb2aeb16b551444787467db42ceab0fc968865af/ruff-0.15.13-py3-none-win_arm64.whl", hash = "sha256:2471da9bd1068c8c064b5fd9c0c4b6dddffd6369cb1cd68b29993b1709ff1b21", size = 11179336, upload-time = "2026-05-14T13:44:33.026Z" }, ] +[[package]] +name = "scikit-learn" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "joblib" }, + { name = "narwhals" }, + { name = "numpy" }, + { name = "scipy" }, + { name = "threadpoolctl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7d/c9a35cf59b20a86fec24d306f1547b78dec194b08d367ce2a3e4854169d9/scikit_learn-1.9.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9656acd4e93f74e0b66c8a36c88830a99252dfa900044d36bc2212ae89a47162", size = 8713289, upload-time = "2026-06-02T11:53:58.788Z" }, + { url = "https://files.pythonhosted.org/packages/3c/a7/552a7821597c632b907f7bfe8f36f9f572777af8ef8a48353041cf8e091a/scikit_learn-1.9.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:24360002ae845e7866522b0a5bbf690802e7bc388cac8663502e78aa98598aa2", size = 8245141, upload-time = "2026-06-02T11:54:01.694Z" }, + { url = "https://files.pythonhosted.org/packages/7d/79/f4a0c4fe9711154cddabf913471153af79056382ddc612cfe5ee0ff4b72e/scikit_learn-1.9.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5162ad10a418c8a282dde04c9aa06965de3e9a65f33c1440c0ae69bb1a09d913", size = 8847671, upload-time = "2026-06-02T11:54:04.448Z" }, + { url = "https://files.pythonhosted.org/packages/f0/af/4d72d9e475ac83719160c662619e4bf7b95c19507cd582e7d0167a3c3dae/scikit_learn-1.9.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fea2cc5677ab49d6f5bade978c866da44957b712d92e9635e8b4f723013c3cb", size = 9118104, upload-time = "2026-06-02T11:54:07.205Z" }, + { url = "https://files.pythonhosted.org/packages/a2/d5/6a58eea2cb9abbb9b3f2bb8b2cfb3243d1152d69f442d256c7af71304769/scikit_learn-1.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:64fa347efc1c839c487433e40c5144d38c336e8a2b59c81aa8660373945c2673", size = 8290674, upload-time = "2026-06-02T11:54:10.087Z" }, + { url = "https://files.pythonhosted.org/packages/65/5b/d4c879cf358f1187141cf90ced473f087183489090244f50c124a2ee478b/scikit_learn-1.9.0-cp314-cp314-win_arm64.whl", hash = "sha256:1b944b6db288f6b926e3650026ddafb988929de95d11fc2cc5fa117773c9ba42", size = 7978807, upload-time = "2026-06-02T11:54:12.769Z" }, + { url = "https://files.pythonhosted.org/packages/8a/43/bfae3121ec67ae09150d453c442c7c1cc166e9aefe056e6ab3b7728a5cfc/scikit_learn-1.9.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:4ccacf04ca5f4b492158a5f28afe0ace43f81b2571e4b9a66d34848b46128949", size = 9031941, upload-time = "2026-06-02T11:54:15.436Z" }, + { url = "https://files.pythonhosted.org/packages/75/b0/20a4546eb17f3b25d3c66df15810411c14ed5065bcfab50b53c96fb627b2/scikit_learn-1.9.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:ee1a8db2c18c08e34c7412d4b10be1cac214cd4ea7dc9715a6a327eb49a37c96", size = 8613528, upload-time = "2026-06-02T11:54:18.842Z" }, + { url = "https://files.pythonhosted.org/packages/18/3c/e440e039bb82cd19004edaaad00acbde0fb9b461083c3ecf37941c557312/scikit_learn-1.9.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:147e9329ef0e39f75d4cffa02b2aa48d827832684926cd5210d9a2cb5c57246b", size = 8855050, upload-time = "2026-06-02T11:54:21.699Z" }, + { url = "https://files.pythonhosted.org/packages/43/26/b341b8dab5998da6270a3a42c2152c578501354d36f944b5856757035ef8/scikit_learn-1.9.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bad8f8b9950321b54c965fdcbac6c6c55e79e16646b49977bcf3668d3870a1a", size = 9097190, upload-time = "2026-06-02T11:54:24.454Z" }, + { url = "https://files.pythonhosted.org/packages/fb/de/b650b4d69b84468cfa2e28a3ff7b8103743029e6446ce1a97fe060ef688c/scikit_learn-1.9.0-cp314-cp314t-win_amd64.whl", hash = "sha256:78fc56eafd4edb9575d2d8950d1dd152061abb573341a1cb7e099fc40f6c6666", size = 8963204, upload-time = "2026-06-02T11:54:27.428Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f3/ff83d76d7418112e5a61326443cdda87be3545dd8d6599c95b2481a4419e/scikit_learn-1.9.0-cp314-cp314t-win_arm64.whl", hash = "sha256:051075bda8b7aab87b1906ab3d4740a1e1224a19d7b3781a576736edc94e76aa", size = 8222661, upload-time = "2026-06-02T11:54:30.192Z" }, +] + +[[package]] +name = "scipy" +version = "1.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b5/915a19b3de2f7430062b509653563db1633ddbb6f021b06731521115d4e2/scipy-1.18.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4c256ee70c0d1a8a2ace807e199ccd4e3f57037433842abb3fb36bc17eaa9578", size = 31036253, upload-time = "2026-06-19T15:00:43.216Z" }, + { url = "https://files.pythonhosted.org/packages/d7/88/b72def7262e150d16be13fca37a96481138d624e700340bc3362a7588929/scipy-1.18.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:2ef3abc54a4ffc53765374b0d5728532dfdd2585ed23f6b11c206a1f0b1b9af8", size = 28673758, upload-time = "2026-06-19T15:00:46.663Z" }, + { url = "https://files.pythonhosted.org/packages/91/02/2e636a61a525632c373cf6a9c24442a3ffb79e364d38e98b32042964ac32/scipy-1.18.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:f2a6af57bd9e4a75d70e4117e78a1bbee84f79ae3fbb6d0111005d6ebcc4cb8d", size = 20415514, upload-time = "2026-06-19T15:00:49.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b6/2135974442f6aba159d9d39d774a1c8cb19947016725d69fecc685df45bf/scipy-1.18.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:3f1ac564d3bf6c03d861d2cd87a1bea0da2887136f7fb1bf519c05a8971452d6", size = 23034398, upload-time = "2026-06-19T15:00:51.941Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e6/ba89ec5abf6ee9257c0d1ec985573f3ae32742c24bc03e016388a40b1b15/scipy-1.18.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40395a5fcd1abee49a5c7aaa98c29db393eedc835138560a588c47ec16156690", size = 33998032, upload-time = "2026-06-19T15:00:54.838Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c4/bc41eb19b0fd0db868f4132920879019318d80cc522ad8f2bca4611af808/scipy-1.18.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ca01e8ae69f1b18e9a58d91afead31be3cef0dd905a10249dac559ee15460a0", size = 35283333, upload-time = "2026-06-19T15:00:58.152Z" }, + { url = "https://files.pythonhosted.org/packages/53/a4/cbdeef6eb3830a8462a9d4ada814de5fc984345cc9ecf17cbec51a036f1e/scipy-1.18.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7a7f3b01647384dbc3a711e8c6778e0aabbe93959249fef5c7393396bcac0867", size = 35610216, upload-time = "2026-06-19T15:01:01.155Z" }, + { url = "https://files.pythonhosted.org/packages/80/4d/b2b82502b65f661d1b789c1665dcdf315d5f12194e06fc0b37946294ebae/scipy-1.18.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6aa94e78ec192a30063a5e72e561c28af769dc311190b24fe91774eff1969709", size = 37418960, upload-time = "2026-06-19T15:01:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/93/3e/902d836831474b0ab5a37d16404f7bc5fafd9efba632890e271ba952635f/scipy-1.18.0-cp314-cp314-win_amd64.whl", hash = "sha256:2d8bbdc6c817f5b4006a54d799d4f5bab6f910193cbb9a1ff310833d4d270f61", size = 37288845, upload-time = "2026-06-19T15:01:07.822Z" }, + { url = "https://files.pythonhosted.org/packages/b6/43/8d73b337a3bdb14daa0314f0434210747c02d79d729ce1777574a817dcf6/scipy-1.18.0-cp314-cp314-win_arm64.whl", hash = "sha256:18e9575f1569b2c54174e6159d32942e03731177f63dce7975f0a0c88d102f5b", size = 24988971, upload-time = "2026-06-19T15:01:11.076Z" }, + { url = "https://files.pythonhosted.org/packages/b4/b4/f11918b0508a2787031a0499a03fbe3546f3bb5ca05d01038c45b278c09a/scipy-1.18.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:f351e0dd702687d12a402b867a1b4146a256923e1c38317cbc472f6372b94707", size = 31399325, upload-time = "2026-06-19T15:01:13.723Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d1/1f287b57c0ff0ee5185dff3946d92c8017d39b0e431f0ae79a3ff1859512/scipy-1.18.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:7c7a51b33ce387193c97f228320cf8e87361daa1bba750638677729598b3e677", size = 29092110, upload-time = "2026-06-19T15:01:16.908Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1a/7b74eb6c392fdcb27d414c0e7558a6d0231eb3b6d73571f479bb81ea8794/scipy-1.18.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:84031d7b052a54fae2f8632e0ec802073d385476eb9a63079bce6e23ef9283d4", size = 20833811, upload-time = "2026-06-19T15:01:20.488Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ad/f3941716320a7b9cb4d68734a903b45fe16eff5fb7da7e16f2e619304979/scipy-1.18.0-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:56abf29a7c067dde59be8b9a22d606a4ea1b2f2a4b756d9d903c62818f5dacce", size = 23396644, upload-time = "2026-06-19T15:01:23.364Z" }, + { url = "https://files.pythonhosted.org/packages/22/22/1446b62ffe07f9719b7d9b1b6a4e05a772833ae8f441fe4c22c34c9b250f/scipy-1.18.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ad44305cfa24b1ba5803cbbebf033590ccbac1aa5d612d727b785325ab408b0", size = 34079318, upload-time = "2026-06-19T15:01:26.002Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/b87da667098bb470fa30c7011b0ba351ee976dd395c78798c66e941665a3/scipy-1.18.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:945c1761b93f38d7f99ae81ae80c63e621471608c7eeead563f6df025585cd58", size = 35324320, upload-time = "2026-06-19T15:01:28.881Z" }, + { url = "https://files.pythonhosted.org/packages/f8/a1/c7932f91909759b0267f75fdea34e91309f96b895757534b76a90b6b4344/scipy-1.18.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1a4441f15d620578772a49e5ab48c0ee1f7a0220e387110283062729136b2553", size = 35699541, upload-time = "2026-06-19T15:01:31.968Z" }, + { url = "https://files.pythonhosted.org/packages/f7/86/5185061a1fcc41d18c5dc2463969b3a3964b31d9ac67b2fb05d4c7ff7670/scipy-1.18.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9aac6192fac56bf2ca534389d24623f07b39ff83317d58287285e7fbd622ff76", size = 37472480, upload-time = "2026-06-19T15:01:35.136Z" }, + { url = "https://files.pythonhosted.org/packages/31/8e/f04c68e39919a010d34f2ee1367fd705b0a25a02f609d755f0bfbc0a15fc/scipy-1.18.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e40baea28ae7f5475c779741e2d90b1247c78531207b49c7030e698ff81cee3f", size = 37365390, upload-time = "2026-06-19T15:01:38.091Z" }, + { url = "https://files.pythonhosted.org/packages/d5/19/969dc072906c84dd0a3b05dcf57ea750936087d7873549e408b35cfc3f97/scipy-1.18.0-cp314-cp314t-win_arm64.whl", hash = "sha256:368e0a705903c466aa5f08eefb39e6b1b6b2d659e7352a31fd9e2438365be0f8", size = 25279661, upload-time = "2026-06-19T15:01:40.817Z" }, +] + [[package]] name = "semver" version = "3.0.4" @@ -2883,6 +3009,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", size = 78154, upload-time = "2019-08-30T21:37:03.543Z" }, ] +[[package]] +name = "threadpoolctl" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, +] + [[package]] name = "toml" version = "0.10.2" @@ -3311,6 +3446,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ed/b5/38fba836844233b961a0026d96f39f893eab63757e7757fd1d16fb02aa80/whenever-0.10.0-py3-none-any.whl", hash = "sha256:70feda454af6b2c231abd428b9430cd75492a000ca1d1edc42976d6fea265eec", size = 119264, upload-time = "2026-04-05T18:43:48.077Z" }, ] +[[package]] +name = "xgboost" +version = "3.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "nvidia-nccl-cu12", marker = "sys_platform == 'linux'" }, + { name = "scipy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/41/846d4de2b8fc694073fd3ac5052caf68caa1ea11cb7fa32d7ad9c049b232/xgboost-3.3.0.tar.gz", hash = "sha256:58bcb8a4cace648cdab7b94fa4f16d2c9ff26d90dd4d26907168106fa06d8746", size = 1224702, upload-time = "2026-06-17T21:26:50.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/72/3b68983c0215ef65d48e9eeb1f168c3c6e3d62a61ece605de3209c79cae1/xgboost-3.3.0-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:07688a377046b8640897b62421150bf73c6cc7101823474ec6ad08b93290f587", size = 2553505, upload-time = "2026-06-17T21:21:32.146Z" }, + { url = "https://files.pythonhosted.org/packages/c9/62/b49e756822b29909d0c95ed334662dc6c7c81a99ec6bc10dc18e69f3d6e7/xgboost-3.3.0-py3-none-macosx_12_0_arm64.whl", hash = "sha256:af7cea10f418b7c251ddc8da440f57bdab2990b5fc9f74a35a92b0f150ea287d", size = 2376040, upload-time = "2026-06-17T21:22:01.981Z" }, + { url = "https://files.pythonhosted.org/packages/47/3a/a0adcd1ee28f525bd5c9dc3ebe78a7599bf97c22866d6449f967b829e338/xgboost-3.3.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:624a83aeb1e7ba081719795db179f4ce6fff12e79de05cd9baf15ee48fd22f0e", size = 98180629, upload-time = "2026-06-17T21:24:00.804Z" }, + { url = "https://files.pythonhosted.org/packages/47/1f/8b3e578cfd8e3bcdb4374e2bbe0b40b4e5320accb5cbdcf535ecc512eb5c/xgboost-3.3.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:f59edaf28eccd1c519788607c72ed907ee6cedfa933d706620bc1612d24b354e", size = 98716607, upload-time = "2026-06-17T21:26:21.058Z" }, + { url = "https://files.pythonhosted.org/packages/07/6b/087fd5d28fdbb90d385c50ee9308a820241b82feebdf42e72e19a48e4b32/xgboost-3.3.0-py3-none-win_amd64.whl", hash = "sha256:b06057f6a018fc04e6b3e0c15568ca636b8151a5b5f333478e500fcaf4fc7594", size = 69522696, upload-time = "2026-06-17T21:20:53.707Z" }, +] + [[package]] name = "zensical" version = "0.0.43" From 3ccfe14ff7687a37f7a4462871adaf8ec3871421 Mon Sep 17 00:00:00 2001 From: Jae Date: Thu, 25 Jun 2026 12:13:45 -0600 Subject: [PATCH 2/5] chore(eta): add placeholder global baseline model for dev/test Deterministic synthetic baseline (polyreg_distance, global) so the stop-time producer runs end-to-end on a fresh checkout without a trained model. Kept as a standalone commit so it is trivial to drop once the retraining suite supplies real models. Regenerate with: python -m gtfs_eta.seed_baseline_model. --- .../polyreg_distance_global_baseline_v0.pkl | Bin 0 -> 946 bytes ...olyreg_distance_global_baseline_v0_meta.json | 16 ++++++++++++++++ backend/eta_models/registry.json | 10 ++++++++++ 3 files changed, 26 insertions(+) create mode 100644 backend/eta_models/polyreg_distance_global_baseline_v0.pkl create mode 100644 backend/eta_models/polyreg_distance_global_baseline_v0_meta.json create mode 100644 backend/eta_models/registry.json diff --git a/backend/eta_models/polyreg_distance_global_baseline_v0.pkl b/backend/eta_models/polyreg_distance_global_baseline_v0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..f5334bf8defa01aa7f68a1d3256f92d15590e409 GIT binary patch literal 946 zcmYjQv2N5r5WP#{%L$Hf3W$QF2?YuT7BmQ?NGXCq;=~bB(V*4Z9^0#|y=HeEaRm|- z35m2_23krgzJVX$7ig%EprAxFFl*mk*kV09J3I50;Lns=(;=TR!Wvm6ERUTPyReB$eM}K@n`;We(t(eN?hBjwr4MKv_N5v zpQW{w{ybz;KI{OrAvoxQ+u^pM4H&nQWF&@J)WD(Xz7jIiPW zyiS^zi7FUPjRhR%>%=066{HN^rLr!nNyuWQAtW0#O#yLEwJM@IO)2FmZoVJ3^D^Im zfB5?1)8UtG;@c#kZ(=7EP_SD_@8ZevTinCV(W2{a;dnfL^p{q%jXD;i(ZV6-YnDRX za@jCqL}|I@K&bk0t|XX!nx1oWDae22f3!s{t>9X7it3q|OYX7eTRZ*E_oqLfM1MZx z5mm@H@@u{`;4WGc!lX(Uq%ZZEO7;pUl%DwFq+X;9>D4}PgX#Q`G}6zr8_xbaW)*3v JD&%z({sUoRbkhI; literal 0 HcmV?d00001 diff --git a/backend/eta_models/polyreg_distance_global_baseline_v0_meta.json b/backend/eta_models/polyreg_distance_global_baseline_v0_meta.json new file mode 100644 index 0000000..628190a --- /dev/null +++ b/backend/eta_models/polyreg_distance_global_baseline_v0_meta.json @@ -0,0 +1,16 @@ +{ + "model_type": "polyreg_distance", + "route_id": null, + "route_specific": false, + "degree": 1, + "alpha": 1.0, + "dataset": "synthetic_constant_speed", + "n_samples": 1000, + "metrics": { + "test_mae_seconds": 30.0, + "test_mae_minutes": 0.5 + }, + "model_key": "polyreg_distance_global_baseline_v0", + "saved_at": "2026-06-25T18:06:35.423137", + "model_path": "polyreg_distance_global_baseline_v0.pkl" +} \ No newline at end of file diff --git a/backend/eta_models/registry.json b/backend/eta_models/registry.json new file mode 100644 index 0000000..749de3d --- /dev/null +++ b/backend/eta_models/registry.json @@ -0,0 +1,10 @@ +{ + "polyreg_distance_global_baseline_v0": { + "model_path": "polyreg_distance_global_baseline_v0.pkl", + "meta_path": "polyreg_distance_global_baseline_v0_meta.json", + "saved_at": "2026-06-25T18:06:35.423137", + "model_type": "polyreg_distance", + "route_id": null, + "dataset": "synthetic_constant_speed" + } +} \ No newline at end of file From 46f226fd0d83bb1e620443f5c7967d33df458782 Mon Sep 17 00:00:00 2001 From: Jae Date: Thu, 25 Jun 2026 12:13:45 -0600 Subject: [PATCH 3/5] feat(eta): generate stop-time updates from ETA predictions Replace the fake_stop_times placeholder with a real producer that calls gtfs_eta.estimate_stop_times. Pure/impure split: compute_stop_time_updates derives contract entries from run state + shape geometry; produce_stop_times does the Redis I/O. Upcoming stops come from the monotonic shape geometry and distances feed the estimator's precomputed-distance hook, fixing duplicate stop_sequences and the non-decreasing upcoming count. Builder sorts/dedups StopTimeUpdate entries defensively. New config: MODEL_REGISTRY_DIR, ETA_MAX_STOPS, ETA_DEFAULT_UNCERTAINTY_S. --- .env.dev | 7 +- .env.example | 8 +- backend/runs/domain/progression/stop_times.py | 319 +++++++++++++++--- backend/schedule_engine/builders.py | 14 +- 4 files changed, 303 insertions(+), 45 deletions(-) diff --git a/.env.dev b/.env.dev index f216c37..2dce194 100644 --- a/.env.dev +++ b/.env.dev @@ -3,4 +3,9 @@ DEBUG=True DJANGO_SERVE_STATIC=True LOG_LEVEL=DEBUG CREATE_SUPERUSER=True -RUN_MIGRATIONS=True \ No newline at end of file +RUN_MIGRATIONS=True + +# ETA model registry (dev default; seed once with seed_baseline_model) +MODEL_REGISTRY_DIR=eta_models +ETA_MAX_STOPS=10 +ETA_DEFAULT_UNCERTAINTY_S=120 \ No newline at end of file diff --git a/.env.example b/.env.example index c14044e..6da2348 100644 --- a/.env.example +++ b/.env.example @@ -54,4 +54,10 @@ FLOWER_DOMAIN=tasks.databus.simovilab.com DOCS_DOMAIN=docs.databus.simovilab.com # Certificate in production -CERT_RESOLVER=letsencrypt \ No newline at end of file +CERT_RESOLVER=letsencrypt + +# ETA model registry (used by gtfs_eta; set to a writable directory) +# Seed the baseline model once: MODEL_REGISTRY_DIR=eta_models ./.venv/bin/python -m gtfs_eta.seed_baseline_model +MODEL_REGISTRY_DIR=eta_models +ETA_MAX_STOPS=3 +ETA_DEFAULT_UNCERTAINTY_S=120 \ No newline at end of file diff --git a/backend/runs/domain/progression/stop_times.py b/backend/runs/domain/progression/stop_times.py index c02eb01..335d974 100644 --- a/backend/runs/domain/progression/stop_times.py +++ b/backend/runs/domain/progression/stop_times.py @@ -1,36 +1,54 @@ -"""Redis glue for the server-side stop-time-updates producer (seam / placeholder). +"""Real ETA stop-time-updates producer — impure/pure split. -This module is the impure counterpart to the pure computation in -``schedule_engine/fake_stop_times.py``. It reads from Redis, delegates to the -existing fake builder, maps the fake output to the typed contract, and writes the -projection back to Redis as a JSON string under ``run::stop_time_updates``. +Pure computation lives in :func:`compute_stop_time_updates`; all Redis I/O +lives in :func:`produce_stop_times`. The split mirrors +``compute.py`` / ``producer.py`` in this package. -Called by ``realtime_engine/mqtt.py`` after every successful position write so -that ``run::stop_time_updates`` is kept current for the GTFS-RT builder. +Called by ``realtime_engine/tasks.py`` after every successful position write +so that ``run::stop_time_updates`` is kept current for the GTFS-RT +builder. -The Redis client mirrors the pattern used in ``producer.py``: module-level client -configured from environment variables. - -Do NOT export ``produce_stop_times`` from ``runs.domain.progression.__init__``. -Import it by full module path:: - - from runs.domain.progression.stop_times import produce_stop_times +Environment variables +--------------------- +MODEL_REGISTRY_DIR + Directory where the ETA model registry is stored. Read by + ``gtfs_eta`` itself from the environment — just set it before + starting the worker. Example: ``eta_models/``. +ETA_MAX_STOPS + Maximum number of upcoming stops to predict. Default: 3. +ETA_DEFAULT_UNCERTAINTY_S + Uncertainty value (seconds) attached to every predicted arrival. + Default: 120. """ import logging import os +from datetime import datetime, timezone import redis -from runs.domain.telemetry import keys, stop_time_updates +from runs.domain.telemetry import keys, position, stop_time_updates, vehicle_stop_status +from runs.domain.progression.geo import project_point_to_polyline +from runs.domain.progression.shapes import ShapeGeometry, get_shape_geometry logger = logging.getLogger(__name__) -# Comfortably above the position-update interval (~1-5 s) so a stalled producer -# expires the projection instead of serving stale arrivals; run lifecycle still -# owns hard cleanup. +# --------------------------------------------------------------------------- +# Module-level config +# --------------------------------------------------------------------------- + +# Comfortably above the position-update interval (~1-5 s) so a stalled +# producer expires the projection instead of serving stale arrivals; run +# lifecycle still owns hard cleanup. STOP_TIME_UPDATES_TTL_S = 60 +# Maximum upcoming stops to pass to the estimator per position tick. +ETA_MAX_STOPS = int(os.getenv("ETA_MAX_STOPS", "3")) + +# Uncertainty (seconds) attached to every prediction. Passed through to the +# GTFS-RT feed; callers (e.g. apps) can use it for confidence UX. +ETA_DEFAULT_UNCERTAINTY_S = int(os.getenv("ETA_DEFAULT_UNCERTAINTY_S", "120")) + r = redis.Redis( host=os.getenv("REDIS_HOST", "state"), port=int(os.getenv("REDIS_PORT", "6379")), @@ -39,45 +57,262 @@ ) -def produce_stop_times(run_id: str, vehicle_id: str) -> None: # noqa: ARG001 - """Derive and write ``run::stop_time_updates`` from the current run state. +# --------------------------------------------------------------------------- +# Pure helper +# --------------------------------------------------------------------------- - Reads the run hash and the current stop-status progression hash from Redis, - delegates to the fake stop-time builder, maps each fake entry to the typed - contract, and writes the JSON projection back to Redis with a staleness TTL. - Returns immediately without writing anything if the run hash is absent (nothing - to derive from). +def compute_stop_time_updates( + *, + run_hash: dict, + position: dict, + stop_status: dict, + geom: ShapeGeometry, + max_stops: int = ETA_MAX_STOPS, + default_uncertainty_s: int = ETA_DEFAULT_UNCERTAINTY_S, +) -> list[dict]: + """Derive stop-time-update contract entries from run state and geometry. + + Pure: no Redis, no ORM, no side effects. The ``geom`` is passed in so + the caller can decide when to skip (and leave last-good in Redis to TTL). + + Parameters + ---------- + run_hash: + Raw Redis hash for ``run:`` — all string values. + position: + Typed position dict as returned by ``position.from_redis``. + Keys: ``latitude``, ``longitude``, optionally ``speed``, ``timestamp``. + stop_status: + Typed stop-status dict as returned by ``vehicle_stop_status.from_redis``. + geom: + Pre-loaded :class:`ShapeGeometry` for the trip's shape. + max_stops: + Maximum number of upcoming stops to predict. + default_uncertainty_s: + Uncertainty (seconds) attached to every predicted arrival. + + Returns + ------- + list[dict] + Zero or more contract dicts, each with exactly the five fields + required by :func:`stop_time_updates.to_redis`: + ``stop_sequence``, ``stop_id``, ``arrival_time``, + ``departure_time``, ``uncertainty``. + Empty list when no predictions are available or the estimator + returns a top-level error. + """ + # ------------------------------------------------------------------ + # 1. Determine current stop sequence and status + # ------------------------------------------------------------------ + current_stop_sequence: int = stop_status.get( + vehicle_stop_status.CURRENT_STOP_SEQUENCE, 0 + ) or 0 + status: str = stop_status.get(vehicle_stop_status.CURRENT_STATUS, "") + + # ------------------------------------------------------------------ + # 2. Project vehicle onto polyline → vehicle_progress_m + # ------------------------------------------------------------------ + lat = position.get("latitude") + lon = position.get("longitude") + if lat is None or lon is None: + return [] + + proj = project_point_to_polyline(float(lat), float(lon), list(geom.polyline)) + vehicle_progress_m: float = proj["progress_m"] + + # ------------------------------------------------------------------ + # 3. Build upcoming_stops list (filter by sequence, compute distances) + # ------------------------------------------------------------------ + upcoming_stops: list[dict] = [] + for stop in geom.stops: + seq: int = stop["stop_sequence"] + if status == "STOPPED_AT": + if seq <= current_stop_sequence: + continue + else: + if seq < current_stop_sequence: + continue + shape_dist = max(0.0, stop["progress_m"] - vehicle_progress_m) + upcoming_stops.append( + { + "stop_id": stop["stop_id"], + "stop_sequence": seq, + "lat": stop["lat"], + "lon": stop["lon"], + "total_stop_sequence": len(geom.stops), + "shape_distance_to_stop": shape_dist, + } + ) + + if not upcoming_stops: + return [] + + # ------------------------------------------------------------------ + # 4. Build vehicle_position dict in estimator contract format + # ------------------------------------------------------------------ + raw_ts = position.get("timestamp") + if raw_ts is not None: + ts_iso = datetime.fromtimestamp(int(raw_ts), tz=timezone.utc).isoformat() + else: + ts_iso = datetime.now(tz=timezone.utc).isoformat() + + vehicle_position = { + "vehicle_id": run_hash.get("vehicle", ""), + "lat": float(lat), + "lon": float(lon), + "speed": float(position.get("speed", 0.0) or 0.0), + "timestamp": ts_iso, + "route": run_hash.get("route_id", ""), + } + + # ------------------------------------------------------------------ + # 5. Build ShapePolyline for the estimator (lazy import → clean startup) + # ------------------------------------------------------------------ + from gtfs_eta.feature_engineering.spatial import ShapePolyline # noqa: PLC0415 + + shape = ShapePolyline([(pt[0], pt[1]) for pt in geom.polyline]) + + # ------------------------------------------------------------------ + # 6. Call estimator (lazy import keeps Django startup clean). + # We import the *module* and call via attribute so tests can patch + # ``gtfs_eta.eta_service.estimator.estimate_stop_times`` reliably. + # ------------------------------------------------------------------ + import gtfs_eta.eta_service.estimator as _estimator_mod # noqa: PLC0415 + + result = _estimator_mod.estimate_stop_times( + vehicle_position, + upcoming_stops, + route_id=run_hash.get("route_id"), + trip_id=run_hash.get("trip_id"), + prefer_route_model=True, + max_stops=max_stops, + shape=shape, + ) + + # Top-level error or empty predictions → safe to return [] + if result.get("error") or not result.get("predictions"): + if result.get("error"): + logger.debug( + "compute_stop_time_updates: estimator error: %s", result["error"] + ) + return [] + + # ------------------------------------------------------------------ + # 7. Output adapter: predictions → contract dicts + # ------------------------------------------------------------------ + entries: list[dict] = [] + for pred in result["predictions"]: + # Skip per-stop failures + if pred.get("error"): + logger.debug( + "compute_stop_time_updates: per-stop error for seq=%s: %s", + pred.get("stop_sequence"), + pred["error"], + ) + continue + eta_ts_str: str | None = pred.get("eta_timestamp") + if not eta_ts_str: + continue + try: + # Handle trailing Z (Python < 3.11 fromisoformat doesn't accept it) + if eta_ts_str.endswith("Z"): + eta_ts_str = eta_ts_str[:-1] + "+00:00" + eta_posix = int(datetime.fromisoformat(eta_ts_str).timestamp()) + except (ValueError, TypeError) as exc: + logger.debug( + "compute_stop_time_updates: bad eta_timestamp %r: %s", eta_ts_str, exc + ) + continue + + entries.append( + { + stop_time_updates.STOP_SEQUENCE: int(pred["stop_sequence"]), + stop_time_updates.STOP_ID: str(pred["stop_id"]), + stop_time_updates.ARRIVAL_TIME: eta_posix, + stop_time_updates.DEPARTURE_TIME: eta_posix, + stop_time_updates.UNCERTAINTY: default_uncertainty_s, + } + ) + + # ------------------------------------------------------------------ + # 8. Dedup by stop_sequence (keep first) + sort ascending + # ------------------------------------------------------------------ + seen: set[int] = set() + deduped: list[dict] = [] + for entry in entries: + seq = entry[stop_time_updates.STOP_SEQUENCE] + if seq not in seen: + seen.add(seq) + deduped.append(entry) + + deduped.sort(key=lambda e: e[stop_time_updates.STOP_SEQUENCE]) + return deduped + + +# --------------------------------------------------------------------------- +# Impure producer (Redis glue) +# --------------------------------------------------------------------------- + + +def produce_stop_times(run_id: str, vehicle_id: str) -> None: + """Derive and write ``run::stop_time_updates`` from current run state. + + 1. Reads the run hash; returns immediately if absent. + 2. Reads the position hash; returns without overwriting if no position. + 3. Reads the stop-status hash. + 4. Resolves shape geometry; returns without overwriting if unavailable + (leaves last-good projection to TTL-expire naturally). + 5. Calls :func:`compute_stop_time_updates`; writes to Redis only when a + non-empty list is returned (no-models case leaves last-good intact). Parameters ---------- run_id: The active run id (string, as stored in ``vehicle::current_run``). vehicle_id: - The vehicle id whose position was just updated (reserved for future use). + The vehicle id whose position was just updated. """ + # Step 1 — run hash run_hash = r.hgetall(keys.run_key(run_id)) if not run_hash: return - prev_raw = r.hgetall(keys.stop_status_key(run_id)) + # Step 2 — position hash (required; exit without overwriting if absent) + from runs.domain.telemetry import position as position_module # noqa: PLC0415 - from schedule_engine.fake_stop_times import build_stop_time_updates + pos_raw = r.hgetall(keys.position_key(vehicle_id)) + if not pos_raw: + return + pos = position_module.from_redis(pos_raw) + if pos.get("latitude") is None or pos.get("longitude") is None: + return - fake_entries = build_stop_time_updates(run=run_hash, progression=prev_raw) + # Step 3 — stop-status hash (tolerate absence) + stop_status_raw = r.hgetall(keys.stop_status_key(run_id)) + stop_status = vehicle_stop_status.from_redis(stop_status_raw) if stop_status_raw else {} - # Map fake entries {stop_sequence, stop_id, eta_posix, uncertainty} - # → contract entries {stop_sequence, stop_id, arrival_time, departure_time, uncertainty} - mapped = [ - { - stop_time_updates.STOP_SEQUENCE: entry["stop_sequence"], - stop_time_updates.STOP_ID: entry["stop_id"], - stop_time_updates.ARRIVAL_TIME: entry["eta_posix"], - stop_time_updates.DEPARTURE_TIME: entry["eta_posix"], - stop_time_updates.UNCERTAINTY: entry["uncertainty"], - } - for entry in fake_entries - ] + # Step 4 — shape geometry (exit without overwriting if unavailable) + shape_id = run_hash.get("shape_id", "") + trip_id = run_hash.get("trip_id", "") + if not shape_id or not trip_id: + return + geom = get_shape_geometry(shape_id, trip_id) + if geom is None: + return + + # Step 5 — compute and conditionally write + entries = compute_stop_time_updates( + run_hash=run_hash, + position=pos, + stop_status=stop_status, + geom=geom, + max_stops=ETA_MAX_STOPS, + default_uncertainty_s=ETA_DEFAULT_UNCERTAINTY_S, + ) + if not entries: + # No predictions (no models trained, etc.) — leave last-good intact. + return - payload = stop_time_updates.to_redis(mapped) + payload = stop_time_updates.to_redis(entries) r.set(keys.stop_time_updates_key(run_id), payload, ex=STOP_TIME_UPDATES_TTL_S) diff --git a/backend/schedule_engine/builders.py b/backend/schedule_engine/builders.py index 8af9acc..70ee6f4 100644 --- a/backend/schedule_engine/builders.py +++ b/backend/schedule_engine/builders.py @@ -201,8 +201,20 @@ def build_trip_update_entity(r, run_id: str) -> dict | None: # entries in the feed). raw = r.get(keys.stop_time_updates_key(run_id)) entries = stop_time_updates.from_redis(raw) + + # Defensive sort + dedup: the producer already guarantees ordering and + # uniqueness, but belt-and-suspenders here ensures a corrupt projection + # never produces an invalid GTFS-RT feed. + seen_seqs: set[int] = set() + deduped_entries: list[dict] = [] + for entry in sorted(entries, key=lambda e: e["stop_sequence"]): + seq = entry["stop_sequence"] + if seq not in seen_seqs: + seen_seqs.add(seq) + deduped_entries.append(entry) + tu["stop_time_update"] = [] - for u in entries: + for u in deduped_entries: tu["stop_time_update"].append( { "stop_sequence": u["stop_sequence"], From 6603390d094b5fdc3e81fbf9a5f84712500f6e0d Mon Sep 17 00:00:00 2001 From: Jae Date: Thu, 25 Jun 2026 12:13:45 -0600 Subject: [PATCH 4/5] test(eta): stop-time producer suite Cover the pure helper, both bug regressions (no duplicate stop_sequence; non-increasing upcoming count), output-adapter edges, and the impure producer's Redis read/skip/write guards. --- .../tests/test_stop_times_producer.py | 784 +++++++++++++++--- 1 file changed, 657 insertions(+), 127 deletions(-) diff --git a/backend/runs/domain/progression/tests/test_stop_times_producer.py b/backend/runs/domain/progression/tests/test_stop_times_producer.py index 4beaf68..76a7ab0 100644 --- a/backend/runs/domain/progression/tests/test_stop_times_producer.py +++ b/backend/runs/domain/progression/tests/test_stop_times_producer.py @@ -1,187 +1,717 @@ -"""Unit tests for produce_stop_times — monkeypatched Redis, no I/O. - -The module-level ``r`` object in stop_times.py is replaced with a MagicMock so -all tests run entirely in-process without a live Redis instance. - -Patch target: ``runs.domain.progression.stop_times.r`` +"""Tests for the real ETA stop-time-updates producer. + +Coverage: +- Pure helper unit tests (ShapeGeometry constructed directly, temp MODEL_REGISTRY_DIR) +- Bug regression: no duplicate stop_sequence; list shrinks as vehicle advances +- Output-adapter edge cases: top-level error, per-stop error, STOPPED_AT +- Impure produce_stop_times: monkeypatched Redis + get_shape_geometry +- Builder hardening: unsorted/duplicate projection is sorted+deduped """ import json +import os +import subprocess +import sys +import tempfile +from datetime import datetime, timezone from unittest.mock import MagicMock, patch import pytest import runs.domain.progression.stop_times as stop_times_module +from runs.domain.progression.shapes import ShapeGeometry, assemble_geometry from runs.domain.progression.stop_times import ( + ETA_DEFAULT_UNCERTAINTY_S, STOP_TIME_UPDATES_TTL_S, + compute_stop_time_updates, produce_stop_times, ) -from runs.domain.telemetry import keys, stop_time_updates +from runs.domain.telemetry import keys, stop_time_updates, vehicle_stop_status +from schedule_engine.builders import build_trip_update_entity # --------------------------------------------------------------------------- -# Helpers +# Shared geometry fixture helpers # --------------------------------------------------------------------------- -VEHICLE_ID = "v-42" -RUN_ID = "run-99" +# A simple straight N–S line: 6 stops spaced ~111 m apart (1 arc-second of lat). +# Shape: 7 points at lon = -84.0, lat from 9.900 to 9.906. +_LAT_BASE = 9.900 +_LON = -84.0 +_D_LAT = 0.001 # ≈ 111 m per step -_RUN_RAW = { - "trip_id": "trip-1", - "route_id": "route-1", - "shape_id": "shape-1", - "vehicle": VEHICLE_ID, -} -_STOP_STATUS_RAW = { - "current_stop_sequence": "3", - "stop_id": "STOP-42", - "current_status": "IN_TRANSIT_TO", -} +def _straight_shape_points(n_points: int = 7) -> list[tuple[float, float, int]]: + """Return (lat, lon, seq) tuples for a straight N–S polyline.""" + return [(_LAT_BASE + i * _D_LAT, _LON, i) for i in range(n_points)] -# Two fake entries that build_stop_time_updates might return -_FAKE_STOP_ENTRIES = [ - { - "stop_sequence": 3, - "stop_id": "stop-1", - "eta_posix": 1700001000, - "uncertainty": 120, - }, - { - "stop_sequence": 4, - "stop_id": "stop-2", - "eta_posix": 1700001300, - "uncertainty": 120, - }, -] - - -def _fake_redis(run_raw=None, stop_status_raw=None) -> MagicMock: - r = MagicMock() - def hgetall_side_effect(key: str) -> dict: - if key == keys.run_key(RUN_ID): - return run_raw if run_raw is not None else _RUN_RAW - if key == keys.stop_status_key(RUN_ID): - return stop_status_raw if stop_status_raw is not None else {} - return {} +def _straight_stop_rows(n_stops: int = 6) -> list[dict]: + """Return stop rows snapped to the first n_stops polyline vertices.""" + return [ + { + "stop_id": f"S{i}", + "stop_sequence": i, + "lat": _LAT_BASE + i * _D_LAT, + "lon": _LON, + } + for i in range(n_stops) + ] - r.hgetall.side_effect = hgetall_side_effect - return r + +def make_straight_geom(n_stops: int = 6) -> ShapeGeometry: + """Build a ShapeGeometry from the straight test polyline.""" + return assemble_geometry( + shape_id="test-shape", + trip_id="test-trip", + shape_points=_straight_shape_points(n_stops + 1), + stop_rows=_straight_stop_rows(n_stops), + ) # --------------------------------------------------------------------------- -# Test 1 — Empty run hash: returns early, no r.set called +# Fixture: seed a baseline model into a temp directory # --------------------------------------------------------------------------- -def test_returns_early_when_run_hash_is_empty(monkeypatch): - fake_r = _fake_redis(run_raw={}) - monkeypatch.setattr(stop_times_module, "r", fake_r) - - produce_stop_times(RUN_ID, VEHICLE_ID) - - fake_r.set.assert_not_called() +@pytest.fixture(scope="module") +def model_dir(): + """Seed the baseline ETA model into a temp directory once per module.""" + with tempfile.TemporaryDirectory() as tmpdir: + env = {**os.environ, "MODEL_REGISTRY_DIR": tmpdir} + result = subprocess.run( + [sys.executable, "-m", "gtfs_eta.seed_baseline_model"], + env=env, + capture_output=True, + text=True, + cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + ) + assert result.returncode == 0, ( + f"seed_baseline_model failed:\nstdout: {result.stdout}\nstderr: {result.stderr}" + ) + yield tmpdir # --------------------------------------------------------------------------- -# Test 2 — Happy path: maps fake entries to contract and writes JSON +# Helper: build a minimal position dict (typed, as from position.from_redis) # --------------------------------------------------------------------------- -def test_happy_path_writes_stop_time_updates_key(monkeypatch): - fake_r = _fake_redis() - monkeypatch.setattr(stop_times_module, "r", fake_r) +def _pos(lat: float, lon: float, speed: float = 4.5, ts: int = 1_700_000_000) -> dict: + return { + "latitude": lat, + "longitude": lon, + "speed": speed, + "timestamp": ts, + } - with patch( - "schedule_engine.fake_stop_times.build_stop_time_updates", - return_value=_FAKE_STOP_ENTRIES, - ): - produce_stop_times(RUN_ID, VEHICLE_ID) - fake_r.set.assert_called_once() - call_args = fake_r.set.call_args - written_key = call_args.args[0] if call_args.args else call_args.kwargs.get("name") - assert written_key == keys.stop_time_updates_key(RUN_ID) +def _run_hash(vehicle: str = "V1", route_id: str = "R1", trip_id: str = "T1") -> dict: + return { + "vehicle": vehicle, + "route_id": route_id, + "trip_id": trip_id, + "shape_id": "test-shape", + } -def test_happy_path_payload_is_valid_json(monkeypatch): - fake_r = _fake_redis() - monkeypatch.setattr(stop_times_module, "r", fake_r) +# --------------------------------------------------------------------------- +# Pure helper — happy path +# --------------------------------------------------------------------------- - with patch( - "schedule_engine.fake_stop_times.build_stop_time_updates", - return_value=_FAKE_STOP_ENTRIES, - ): - produce_stop_times(RUN_ID, VEHICLE_ID) - payload_str = fake_r.set.call_args.args[1] - parsed = json.loads(payload_str) - assert isinstance(parsed, list) - assert len(parsed) == 2 +class TestComputeStopTimeUpdatesHappyPath: + """Predictions map correctly; ARRIVAL_TIME is int POSIX; ETAs increase.""" + + def test_returns_list_of_dicts(self, model_dir): + geom = make_straight_geom() + # Vehicle at the very start of the route (before stop 0) + pos = _pos(_LAT_BASE - 0.0005, _LON) + with patch.dict(os.environ, {"MODEL_REGISTRY_DIR": model_dir}): + result = compute_stop_time_updates( + run_hash=_run_hash(), + position=pos, + stop_status={}, + geom=geom, + max_stops=3, + default_uncertainty_s=120, + ) + assert isinstance(result, list) + if result: # may be empty if model not loaded but seed should work + assert all(isinstance(e, dict) for e in result) + + def test_arrival_time_is_int_posix(self, model_dir): + geom = make_straight_geom() + pos = _pos(_LAT_BASE - 0.0005, _LON) + with patch.dict(os.environ, {"MODEL_REGISTRY_DIR": model_dir}): + result = compute_stop_time_updates( + run_hash=_run_hash(), + position=pos, + stop_status={}, + geom=geom, + max_stops=3, + default_uncertainty_s=120, + ) + for entry in result: + assert isinstance(entry[stop_time_updates.ARRIVAL_TIME], int) + assert isinstance(entry[stop_time_updates.DEPARTURE_TIME], int) + # Sanity: POSIX in a plausible range (year 2000 → 2100) + assert 946_684_800 < entry[stop_time_updates.ARRIVAL_TIME] < 4_102_444_800 + + def test_uncertainty_equals_default(self, model_dir): + geom = make_straight_geom() + pos = _pos(_LAT_BASE - 0.0005, _LON) + with patch.dict(os.environ, {"MODEL_REGISTRY_DIR": model_dir}): + result = compute_stop_time_updates( + run_hash=_run_hash(), + position=pos, + stop_status={}, + geom=geom, + max_stops=3, + default_uncertainty_s=99, + ) + for entry in result: + assert entry[stop_time_updates.UNCERTAINTY] == 99 + + def test_etas_are_non_decreasing(self, model_dir): + """ETAs must increase (or stay equal) as distance increases.""" + geom = make_straight_geom() + pos = _pos(_LAT_BASE - 0.0005, _LON) + with patch.dict(os.environ, {"MODEL_REGISTRY_DIR": model_dir}): + result = compute_stop_time_updates( + run_hash=_run_hash(), + position=pos, + stop_status={}, + geom=geom, + max_stops=6, + default_uncertainty_s=120, + ) + times = [e[stop_time_updates.ARRIVAL_TIME] for e in result] + for i in range(1, len(times)): + assert times[i] >= times[i - 1], ( + f"ETA decreased: entry[{i - 1}]={times[i - 1]} > entry[{i}]={times[i]}" + ) + + def test_stop_id_and_sequence_match_geom(self, model_dir): + geom = make_straight_geom(n_stops=6) + pos = _pos(_LAT_BASE - 0.0005, _LON) + with patch.dict(os.environ, {"MODEL_REGISTRY_DIR": model_dir}): + result = compute_stop_time_updates( + run_hash=_run_hash(), + position=pos, + stop_status={}, + geom=geom, + max_stops=3, + default_uncertainty_s=120, + ) + geom_stop_ids = {s["stop_id"] for s in geom.stops} + for entry in result: + assert entry[stop_time_updates.STOP_ID] in geom_stop_ids -def test_fake_to_contract_mapping(monkeypatch): - """Each fake entry {eta_posix} must map to contract {arrival_time, departure_time}.""" - fake_r = _fake_redis() - monkeypatch.setattr(stop_times_module, "r", fake_r) +# --------------------------------------------------------------------------- +# Bug regression: no duplicates; list shrinks as vehicle advances +# --------------------------------------------------------------------------- - with patch( - "schedule_engine.fake_stop_times.build_stop_time_updates", - return_value=_FAKE_STOP_ENTRIES, - ): - produce_stop_times(RUN_ID, VEHICLE_ID) - payload_str = fake_r.set.call_args.args[1] - entries = stop_time_updates.from_redis(payload_str) - assert len(entries) == 2 - # Verify the eta_posix → arrival_time / departure_time mapping - assert entries[0][stop_time_updates.ARRIVAL_TIME] == 1700001000 - assert entries[0][stop_time_updates.DEPARTURE_TIME] == 1700001000 - assert entries[0][stop_time_updates.STOP_SEQUENCE] == 3 - assert entries[0][stop_time_updates.STOP_ID] == "stop-1" - assert entries[0][stop_time_updates.UNCERTAINTY] == 120 - - -def test_writes_with_ttl(monkeypatch): - """r.set must be called with ex=STOP_TIME_UPDATES_TTL_S.""" - fake_r = _fake_redis() - monkeypatch.setattr(stop_times_module, "r", fake_r) - - with patch( - "schedule_engine.fake_stop_times.build_stop_time_updates", - return_value=_FAKE_STOP_ENTRIES, - ): - produce_stop_times(RUN_ID, VEHICLE_ID) +class TestBugRegressions: + def test_no_duplicate_stop_sequence_in_output(self, model_dir): + """Output must never contain duplicate stop_sequence values.""" + geom = make_straight_geom(n_stops=6) + pos = _pos(_LAT_BASE - 0.0005, _LON) + with patch.dict(os.environ, {"MODEL_REGISTRY_DIR": model_dir}): + result = compute_stop_time_updates( + run_hash=_run_hash(), + position=pos, + stop_status={}, + geom=geom, + max_stops=6, + default_uncertainty_s=120, + ) + seqs = [e[stop_time_updates.STOP_SEQUENCE] for e in result] + assert len(seqs) == len(set(seqs)), f"Duplicate sequences: {seqs}" + + def test_list_length_non_increasing_as_sequence_advances(self, model_dir): + """As current_stop_sequence advances 0→3→5, the result length must not grow.""" + geom = make_straight_geom(n_stops=6) + # Vehicle mid-route + pos = _pos(_LAT_BASE + 2 * _D_LAT + 0.0005, _LON) + + def _count(current_seq: int, status: str = "IN_TRANSIT_TO") -> int: + ss = { + vehicle_stop_status.CURRENT_STOP_SEQUENCE: current_seq, + vehicle_stop_status.CURRENT_STATUS: status, + } + with patch.dict(os.environ, {"MODEL_REGISTRY_DIR": model_dir}): + res = compute_stop_time_updates( + run_hash=_run_hash(), + position=pos, + stop_status=ss, + geom=geom, + max_stops=6, + default_uncertainty_s=120, + ) + return len(res) + + c0 = _count(0) + c3 = _count(3) + c5 = _count(5) + assert c0 >= c3 >= c5, ( + f"List grew as sequence advanced: seq0={c0}, seq3={c3}, seq5={c5}" + ) + + def test_output_contains_only_sequences_gte_current(self, model_dir): + """All returned sequences must be >= current_stop_sequence.""" + geom = make_straight_geom(n_stops=6) + pos = _pos(_LAT_BASE, _LON) + current_seq = 3 + ss = { + vehicle_stop_status.CURRENT_STOP_SEQUENCE: current_seq, + vehicle_stop_status.CURRENT_STATUS: "IN_TRANSIT_TO", + } + with patch.dict(os.environ, {"MODEL_REGISTRY_DIR": model_dir}): + result = compute_stop_time_updates( + run_hash=_run_hash(), + position=pos, + stop_status=ss, + geom=geom, + max_stops=6, + default_uncertainty_s=120, + ) + for entry in result: + assert entry[stop_time_updates.STOP_SEQUENCE] >= current_seq - call_kwargs = fake_r.set.call_args.kwargs - assert call_kwargs.get("ex") == STOP_TIME_UPDATES_TTL_S + +# --------------------------------------------------------------------------- +# Output-adapter edge cases +# --------------------------------------------------------------------------- + + +class TestOutputAdapterEdgeCases: + def test_top_level_error_returns_empty(self): + """When estimator returns a top-level error, result must be [].""" + geom = make_straight_geom() + pos = _pos(_LAT_BASE, _LON) + error_result = { + "predictions": [], + "error": "No trained models found for model_type", + "model_key": None, + } + # estimate_stop_times is lazily imported inside compute_stop_time_updates; + # patch the function in its source module (gtfs_eta.eta_service.estimator). + with patch( + "gtfs_eta.eta_service.estimator.estimate_stop_times", + return_value=error_result, + ): + result = compute_stop_time_updates( + run_hash=_run_hash(), + position=pos, + stop_status={}, + geom=geom, + ) + assert result == [] + + def test_per_stop_error_is_skipped(self): + """A prediction with per-stop error must be excluded from the output.""" + geom = make_straight_geom(n_stops=3) + pos = _pos(_LAT_BASE - 0.0005, _LON) + now_ts = datetime.now(tz=timezone.utc) + estimator_result = { + "predictions": [ + { + "stop_id": "S0", + "stop_sequence": 0, + "distance_to_stop_m": 50.0, + "eta_seconds": None, + "eta_minutes": None, + "eta_formatted": None, + "eta_timestamp": None, + "error": "prediction failed", + }, + { + "stop_id": "S1", + "stop_sequence": 1, + "distance_to_stop_m": 160.0, + "eta_seconds": 36.0, + "eta_minutes": 0.6, + "eta_formatted": "0m 36s", + "eta_timestamp": now_ts.isoformat(), + }, + ], + } + with patch( + "gtfs_eta.eta_service.estimator.estimate_stop_times", + return_value=estimator_result, + ): + result = compute_stop_time_updates( + run_hash=_run_hash(), + position=pos, + stop_status={}, + geom=geom, + ) + # Only S1 should be in the output (S0 has an error) + assert len(result) == 1 + assert result[0][stop_time_updates.STOP_ID] == "S1" + + def test_stopped_at_excludes_current_stop(self, model_dir): + """When status is STOPPED_AT, the current stop must not appear in output.""" + geom = make_straight_geom(n_stops=6) + pos = _pos(_LAT_BASE + 2 * _D_LAT, _LON) + current_seq = 2 + ss = { + vehicle_stop_status.CURRENT_STOP_SEQUENCE: current_seq, + vehicle_stop_status.CURRENT_STATUS: "STOPPED_AT", + } + with patch.dict(os.environ, {"MODEL_REGISTRY_DIR": model_dir}): + result = compute_stop_time_updates( + run_hash=_run_hash(), + position=pos, + stop_status=ss, + geom=geom, + max_stops=6, + default_uncertainty_s=120, + ) + seqs = [e[stop_time_updates.STOP_SEQUENCE] for e in result] + assert current_seq not in seqs, ( + f"STOPPED_AT: current_seq={current_seq} found in output seqs {seqs}" + ) + for seq in seqs: + assert seq > current_seq + + def test_empty_upcoming_stops_returns_empty(self): + """When no upcoming stops remain, result is [].""" + geom = make_straight_geom(n_stops=3) + pos = _pos(_LAT_BASE, _LON) + ss = { + # All stops are behind the vehicle (seq 100 > any stop) + vehicle_stop_status.CURRENT_STOP_SEQUENCE: 100, + vehicle_stop_status.CURRENT_STATUS: "IN_TRANSIT_TO", + } + result = compute_stop_time_updates( + run_hash=_run_hash(), + position=pos, + stop_status=ss, + geom=geom, + ) + assert result == [] + + def test_empty_predictions_returns_empty(self): + """When estimator returns empty predictions list (no error), result is [].""" + geom = make_straight_geom(n_stops=3) + pos = _pos(_LAT_BASE - 0.0005, _LON) + with patch( + "gtfs_eta.eta_service.estimator.estimate_stop_times", + return_value={"predictions": []}, + ): + result = compute_stop_time_updates( + run_hash=_run_hash(), + position=pos, + stop_status={}, + geom=geom, + ) + assert result == [] # --------------------------------------------------------------------------- -# Test 3 — Empty fake entries: writes empty JSON array +# Impure produce_stop_times tests (monkeypatched Redis) # --------------------------------------------------------------------------- +VEHICLE_ID = "V-test" +RUN_ID = "run-test" + +_RUN_RAW = { + "trip_id": "T1", + "route_id": "R1", + "shape_id": "shape-1", + "vehicle": VEHICLE_ID, +} + +_POS_RAW = { + "latitude": str(_LAT_BASE), + "longitude": str(_LON), + "speed": "4.5", + "timestamp": "1700000000", +} + +_STOP_STATUS_RAW = { + "current_stop_sequence": "2", + "stop_id": "S2", + "current_status": "IN_TRANSIT_TO", +} + + +def _fake_redis(run_raw=None, pos_raw=None, stop_status_raw=None) -> MagicMock: + r = MagicMock() + + def hgetall_side_effect(key: str) -> dict: + if key == keys.run_key(RUN_ID): + return run_raw if run_raw is not None else _RUN_RAW + if key == keys.position_key(VEHICLE_ID): + return pos_raw if pos_raw is not None else _POS_RAW + if key == keys.stop_status_key(RUN_ID): + return stop_status_raw if stop_status_raw is not None else _STOP_STATUS_RAW + return {} + + r.hgetall.side_effect = hgetall_side_effect + return r -def test_empty_fake_entries_writes_empty_array(monkeypatch): - fake_r = _fake_redis() - monkeypatch.setattr(stop_times_module, "r", fake_r) - with patch( - "schedule_engine.fake_stop_times.build_stop_time_updates", - return_value=[], - ): +class TestProduceStopTimesImpure: + def test_returns_early_when_run_hash_empty(self, monkeypatch): + fake_r = _fake_redis(run_raw={}) + monkeypatch.setattr(stop_times_module, "r", fake_r) produce_stop_times(RUN_ID, VEHICLE_ID) + fake_r.set.assert_not_called() - fake_r.set.assert_called_once() - payload_str = fake_r.set.call_args.args[1] - assert json.loads(payload_str) == [] + def test_returns_early_when_no_position(self, monkeypatch): + fake_r = _fake_redis(pos_raw={}) + monkeypatch.setattr(stop_times_module, "r", fake_r) + produce_stop_times(RUN_ID, VEHICLE_ID) + fake_r.set.assert_not_called() + + def test_does_not_write_when_no_shape(self, monkeypatch): + fake_r = _fake_redis() + monkeypatch.setattr(stop_times_module, "r", fake_r) + with patch( + "runs.domain.progression.stop_times.get_shape_geometry", + return_value=None, + ): + produce_stop_times(RUN_ID, VEHICLE_ID) + fake_r.set.assert_not_called() + + def test_does_not_write_when_compute_returns_empty(self, monkeypatch): + geom = make_straight_geom() + fake_r = _fake_redis() + monkeypatch.setattr(stop_times_module, "r", fake_r) + with ( + patch( + "runs.domain.progression.stop_times.get_shape_geometry", + return_value=geom, + ), + patch( + "runs.domain.progression.stop_times.compute_stop_time_updates", + return_value=[], + ), + ): + produce_stop_times(RUN_ID, VEHICLE_ID) + fake_r.set.assert_not_called() + + def test_writes_correct_key_with_ttl(self, monkeypatch): + geom = make_straight_geom() + fake_r = _fake_redis() + monkeypatch.setattr(stop_times_module, "r", fake_r) + + now_ts = int(datetime.now(tz=timezone.utc).timestamp()) + fake_entries = [ + { + stop_time_updates.STOP_SEQUENCE: 2, + stop_time_updates.STOP_ID: "S2", + stop_time_updates.ARRIVAL_TIME: now_ts + 60, + stop_time_updates.DEPARTURE_TIME: now_ts + 60, + stop_time_updates.UNCERTAINTY: 120, + }, + ] + with ( + patch( + "runs.domain.progression.stop_times.get_shape_geometry", + return_value=geom, + ), + patch( + "runs.domain.progression.stop_times.compute_stop_time_updates", + return_value=fake_entries, + ), + ): + produce_stop_times(RUN_ID, VEHICLE_ID) + + fake_r.set.assert_called_once() + call_args = fake_r.set.call_args + written_key = call_args.args[0] if call_args.args else call_args.kwargs.get("name") + assert written_key == keys.stop_time_updates_key(RUN_ID) + assert call_args.kwargs.get("ex") == STOP_TIME_UPDATES_TTL_S + + def test_written_payload_parses_to_unique_ascending_sequences(self, monkeypatch): + geom = make_straight_geom() + fake_r = _fake_redis() + monkeypatch.setattr(stop_times_module, "r", fake_r) + + now_ts = int(datetime.now(tz=timezone.utc).timestamp()) + fake_entries = [ + { + stop_time_updates.STOP_SEQUENCE: 2, + stop_time_updates.STOP_ID: "S2", + stop_time_updates.ARRIVAL_TIME: now_ts + 60, + stop_time_updates.DEPARTURE_TIME: now_ts + 60, + stop_time_updates.UNCERTAINTY: 120, + }, + { + stop_time_updates.STOP_SEQUENCE: 3, + stop_time_updates.STOP_ID: "S3", + stop_time_updates.ARRIVAL_TIME: now_ts + 120, + stop_time_updates.DEPARTURE_TIME: now_ts + 120, + stop_time_updates.UNCERTAINTY: 120, + }, + ] + with ( + patch( + "runs.domain.progression.stop_times.get_shape_geometry", + return_value=geom, + ), + patch( + "runs.domain.progression.stop_times.compute_stop_time_updates", + return_value=fake_entries, + ), + ): + produce_stop_times(RUN_ID, VEHICLE_ID) + + payload_str = fake_r.set.call_args.args[1] + parsed = stop_time_updates.from_redis(payload_str) + seqs = [e["stop_sequence"] for e in parsed] + assert len(seqs) == len(set(seqs)), f"Duplicate sequences: {seqs}" + assert seqs == sorted(seqs), f"Not ascending: {seqs}" # --------------------------------------------------------------------------- -# Test 4 — TTL constant is 60 +# Builder hardening: unsorted/duplicate projection is sorted+deduped # --------------------------------------------------------------------------- -def test_ttl_constant_is_60(): - assert STOP_TIME_UPDATES_TTL_S == 60 +class FakeRedis: + """Minimal dict-backed Redis stub for builder tests.""" + + def __init__(self, data: dict): + self._data = data + + def hgetall(self, key: str) -> dict: + val = self._data.get(key, {}) + return dict(val) if isinstance(val, dict) else {} + + def smembers(self, key: str) -> set: + val = self._data.get(key, set()) + return set(val) if isinstance(val, set) else set() + + def get(self, key: str) -> str | None: + val = self._data.get(key) + return val if isinstance(val, str) else None + + +_BUILDER_RUN_ID = "run-b1" +_BUILDER_VID = "V-b1" + + +def _builder_redis_data(projection_entries: list[dict]) -> dict: + return { + "runs:in_progress": {_BUILDER_RUN_ID}, + f"run:{_BUILDER_RUN_ID}": { + "vehicle": _BUILDER_VID, + "trip_id": "trip-x", + "route_id": "route-x", + "schedule_relationship": "SCHEDULED", + }, + f"run:{_BUILDER_RUN_ID}:trip": { + "trip_id": "trip-x", + "route_id": "route-x", + "schedule_relationship": "SCHEDULED", + }, + f"vehicle:{_BUILDER_VID}:position": { + "latitude": "9.900", + "longitude": "-84.0", + "timestamp": "1700000000", + }, + f"vehicle:{_BUILDER_VID}:metadata": { + "id": _BUILDER_VID, + "label": "Bus B1", + }, + f"run:{_BUILDER_RUN_ID}:vehicle_stop_status": { + "current_stop_sequence": "1", + "current_status": "IN_TRANSIT_TO", + }, + keys.stop_time_updates_key(_BUILDER_RUN_ID): stop_time_updates.to_redis( + projection_entries + ), + } + + +class TestBuilderHardening: + def test_unsorted_projection_is_sorted_ascending(self): + now = int(datetime.now(tz=timezone.utc).timestamp()) + # Deliberately reverse order: seq 5 before seq 2 + entries = [ + { + stop_time_updates.STOP_SEQUENCE: 5, + stop_time_updates.STOP_ID: "S5", + stop_time_updates.ARRIVAL_TIME: now + 200, + stop_time_updates.DEPARTURE_TIME: now + 200, + stop_time_updates.UNCERTAINTY: 120, + }, + { + stop_time_updates.STOP_SEQUENCE: 2, + stop_time_updates.STOP_ID: "S2", + stop_time_updates.ARRIVAL_TIME: now + 100, + stop_time_updates.DEPARTURE_TIME: now + 100, + stop_time_updates.UNCERTAINTY: 120, + }, + ] + r = FakeRedis(_builder_redis_data(entries)) + entity = build_trip_update_entity(r, _BUILDER_RUN_ID) + assert entity is not None + updates = entity["trip_update"]["stop_time_update"] + seqs = [u["stop_sequence"] for u in updates] + assert seqs == sorted(seqs), f"Not sorted: {seqs}" + + def test_duplicate_sequences_are_deduped(self): + now = int(datetime.now(tz=timezone.utc).timestamp()) + # Two entries with same stop_sequence=3 + entries = [ + { + stop_time_updates.STOP_SEQUENCE: 3, + stop_time_updates.STOP_ID: "S3-a", + stop_time_updates.ARRIVAL_TIME: now + 100, + stop_time_updates.DEPARTURE_TIME: now + 100, + stop_time_updates.UNCERTAINTY: 120, + }, + { + stop_time_updates.STOP_SEQUENCE: 3, + stop_time_updates.STOP_ID: "S3-b", + stop_time_updates.ARRIVAL_TIME: now + 110, + stop_time_updates.DEPARTURE_TIME: now + 110, + stop_time_updates.UNCERTAINTY: 120, + }, + { + stop_time_updates.STOP_SEQUENCE: 5, + stop_time_updates.STOP_ID: "S5", + stop_time_updates.ARRIVAL_TIME: now + 200, + stop_time_updates.DEPARTURE_TIME: now + 200, + stop_time_updates.UNCERTAINTY: 120, + }, + ] + r = FakeRedis(_builder_redis_data(entries)) + entity = build_trip_update_entity(r, _BUILDER_RUN_ID) + assert entity is not None + updates = entity["trip_update"]["stop_time_update"] + seqs = [u["stop_sequence"] for u in updates] + assert len(seqs) == len(set(seqs)), f"Duplicates remain: {seqs}" + assert len(seqs) == 2 # seq 3 (first kept) + seq 5 + + def test_first_of_duplicate_is_kept(self): + """When two entries share a stop_sequence, the first (lower arrival) is kept.""" + now = int(datetime.now(tz=timezone.utc).timestamp()) + entries = [ + { + stop_time_updates.STOP_SEQUENCE: 3, + stop_time_updates.STOP_ID: "S3-first", + stop_time_updates.ARRIVAL_TIME: now + 100, + stop_time_updates.DEPARTURE_TIME: now + 100, + stop_time_updates.UNCERTAINTY: 120, + }, + { + stop_time_updates.STOP_SEQUENCE: 3, + stop_time_updates.STOP_ID: "S3-second", + stop_time_updates.ARRIVAL_TIME: now + 110, + stop_time_updates.DEPARTURE_TIME: now + 110, + stop_time_updates.UNCERTAINTY: 120, + }, + ] + r = FakeRedis(_builder_redis_data(entries)) + entity = build_trip_update_entity(r, _BUILDER_RUN_ID) + updates = entity["trip_update"]["stop_time_update"] + assert updates[0]["stop_id"] == "S3-first" From 064e6437c1a9ef9fa0def25938d8e62dd66015a7 Mon Sep 17 00:00:00 2001 From: Jae Date: Thu, 25 Jun 2026 12:13:45 -0600 Subject: [PATCH 5/5] chore(eta): drop fake_stop_times placeholder Superseded by the real ETA producer. Removes the fabricated stop-time generator and its static route_stops.csv (whose 0-based sequences surfaced an off-by-one). --- .../schedule_engine/aux_files/route_stops.csv | 61 ------- backend/schedule_engine/fake_stop_times.py | 163 ------------------ 2 files changed, 224 deletions(-) delete mode 100644 backend/schedule_engine/aux_files/route_stops.csv delete mode 100644 backend/schedule_engine/fake_stop_times.py diff --git a/backend/schedule_engine/aux_files/route_stops.csv b/backend/schedule_engine/aux_files/route_stops.csv deleted file mode 100644 index a04bad2..0000000 --- a/backend/schedule_engine/aux_files/route_stops.csv +++ /dev/null @@ -1,61 +0,0 @@ -route_id,shape_id,direction_id,stop_id,stop_sequence,timepoint -bUCR_L2,desde_educacion_sin_milla,0,UCR_0_00,0,1 -bUCR_L2,desde_educacion_sin_milla,0,UCR_0_04,1,0 -bUCR_L2,desde_educacion_sin_milla,0,UCR_0_05,2,0 -bUCR_L2,desde_educacion_sin_milla,0,UCR_0_06,3,0 -bUCR_L2,desde_educacion_sin_milla,0,UCR_0_07,4,0 -bUCR_L2,desde_educacion_sin_milla,0,UCR_0_08,5,0 -bUCR_L2,desde_educacion_sin_milla,0,UCR_0_09,6,0 -bUCR_L2,desde_educacion_sin_milla,0,UCR_0_10,7,0 -bUCR_L2,desde_educacion_sin_milla,0,UCR_0_11,8,0 -bUCR_L1,desde_educacion_con_milla,0,UCR_0_00,0,1 -bUCR_L1,desde_educacion_con_milla,0,UCR_0_02,1,0 -bUCR_L1,desde_educacion_con_milla,0,UCR_0_03,2,0 -bUCR_L1,desde_educacion_con_milla,0,UCR_0_04,3,0 -bUCR_L1,desde_educacion_con_milla,0,UCR_0_05,4,0 -bUCR_L1,desde_educacion_con_milla,0,UCR_0_06,5,0 -bUCR_L1,desde_educacion_con_milla,0,UCR_0_07,6,0 -bUCR_L1,desde_educacion_con_milla,0,UCR_0_08,7,0 -bUCR_L1,desde_educacion_con_milla,0,UCR_0_09,8,0 -bUCR_L1,desde_educacion_con_milla,0,UCR_0_10,9,0 -bUCR_L1,desde_educacion_con_milla,0,UCR_0_11,10,0 -bUCR_L2,desde_artes_sin_milla,0,UCR_0_01,0,1 -bUCR_L2,desde_artes_sin_milla,0,UCR_0_04,1,0 -bUCR_L2,desde_artes_sin_milla,0,UCR_0_05,2,0 -bUCR_L2,desde_artes_sin_milla,0,UCR_0_06,3,0 -bUCR_L2,desde_artes_sin_milla,0,UCR_0_07,4,0 -bUCR_L2,desde_artes_sin_milla,0,UCR_0_08,5,0 -bUCR_L2,desde_artes_sin_milla,0,UCR_0_09,6,0 -bUCR_L2,desde_artes_sin_milla,0,UCR_0_10,7,0 -bUCR_L2,desde_artes_sin_milla,0,UCR_0_11,8,0 -bUCR_L1,desde_artes_con_milla,0,UCR_0_01,0,1 -bUCR_L1,desde_artes_con_milla,0,UCR_0_02,1,0 -bUCR_L1,desde_artes_con_milla,0,UCR_0_03,2,0 -bUCR_L1,desde_artes_con_milla,0,UCR_0_04,3,0 -bUCR_L1,desde_artes_con_milla,0,UCR_0_05,4,0 -bUCR_L1,desde_artes_con_milla,0,UCR_0_06,5,0 -bUCR_L1,desde_artes_con_milla,0,UCR_0_07,6,0 -bUCR_L1,desde_artes_con_milla,0,UCR_0_08,7,0 -bUCR_L1,desde_artes_con_milla,0,UCR_0_09,8,0 -bUCR_L1,desde_artes_con_milla,0,UCR_0_10,9,0 -bUCR_L1,desde_artes_con_milla,0,UCR_0_11,10,0 -bUCR_L1,hacia_educacion,1,UCR_1_00,0,1 -bUCR_L1,hacia_educacion,1,UCR_1_01,1,0 -bUCR_L1,hacia_educacion,1,UCR_1_02,2,0 -bUCR_L1,hacia_educacion,1,UCR_1_03,3,0 -bUCR_L1,hacia_educacion,1,UCR_1_04,4,0 -bUCR_L1,hacia_educacion,1,UCR_1_05,5,0 -bUCR_L1,hacia_educacion,1,UCR_1_06,6,0 -bUCR_L1,hacia_educacion,1,UCR_1_07,7,0 -bUCR_L1,hacia_educacion,1,UCR_1_08,8,0 -bUCR_L1,hacia_educacion,1,UCR_1_09,9,0 -bUCR_L1,hacia_artes,1,UCR_1_00,0,1 -bUCR_L1,hacia_artes,1,UCR_1_01,1,0 -bUCR_L1,hacia_artes,1,UCR_1_02,2,0 -bUCR_L1,hacia_artes,1,UCR_1_03,3,0 -bUCR_L1,hacia_artes,1,UCR_1_04,4,0 -bUCR_L1,hacia_artes,1,UCR_1_05,5,0 -bUCR_L1,hacia_artes,1,UCR_1_06,6,0 -bUCR_L1,hacia_artes,1,UCR_1_07,7,0 -bUCR_L1,hacia_artes,1,UCR_1_08,8,0 -bUCR_L1,hacia_artes,1,UCR_1_10,10,0 \ No newline at end of file diff --git a/backend/schedule_engine/fake_stop_times.py b/backend/schedule_engine/fake_stop_times.py deleted file mode 100644 index fa26369..0000000 --- a/backend/schedule_engine/fake_stop_times.py +++ /dev/null @@ -1,163 +0,0 @@ -# For the _fake_stop_times method (temporary!) -import logging -import random -from datetime import datetime, timedelta -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd - -logger = logging.getLogger(__name__) - -_CSV_FILE_PATH = Path(__file__).resolve().parent / "aux_files" / "route_stops.csv" -# Time in seconds -_UNCERTAINTY_S = 120 -_TIME_OFFSET_MIN_S = 150 -_TIME_OFFSET_MAX_S = 300 -_ARRIVAL_MAX_MIN = 5 - - -def _load_route_stops(csv_file_path) -> pd.DataFrame: - """Load route stops from a CSV file. - - Parameters: - csv_file_path: Name of CSV file with route stops. - - Returns: - pd.DataFrame: Information of CSV file as a Pandas DataFrame. - """ - return pd.read_csv(csv_file_path, dtype={"stop_sequence": np.uint32}) - - -def _generate_stop_entry( - arrival_time, stop_sequence, stop_id, uncertainty -) -> dict[str, Any]: - """Generate a stop entry with given parameters. - - Parameters: - arrival_time: Estimated time of arrival to stop as absolute time. In POSIX time. - stop_sequence: Order of stops in route. - stop_id: ID of stop. - uncertainty: Margin of error in the estimated time of arrival. - - Returns: - dict[str, Any]: A dictionary entry with stop time updates. - """ - return { - "stop_sequence": int(stop_sequence), - "stop_id": str(stop_id), - "eta_posix": int(arrival_time.timestamp()), - "uncertainty": uncertainty, - } - - -def _safe_int(value, default: int = -1) -> int: - """Coerce a Redis-string value to int, returning ``default`` on failure.""" - try: - return int(value) - except (TypeError, ValueError): - return default - - -def build_stop_time_updates(run, progression) -> list[dict[str, Any]]: - """Generate fake stop times for the given run. - - Parameters: - run: Mapping with at least ``route_id`` and ``shape_id`` keys - (typically a Redis hash dict). - progression: Mapping with ``current_stop_sequence`` and - ``current_status`` keys (typically a Redis hash dict). - - Returns: - list[dict[str, Any]]: A list of dictionaries with stop time updates. - """ - stop_time_update: list[dict[str, Any]] = [] - - run = run or {} - progression = progression or {} - - route_id = str(run.get("route_id") or "").strip() - shape_id = str(run.get("shape_id") or "").strip() - if not route_id: - logger.warning("build_stop_time_updates: run missing route_id (run=%s)", run) - return stop_time_update - - try: - route_stops = _load_route_stops(csv_file_path=_CSV_FILE_PATH) - except FileNotFoundError: - logger.exception("Route stops CSV not found at %s", _CSV_FILE_PATH) - return stop_time_update - - # Primary match: route_id AND shape_id - filtered_stops = route_stops[ - (route_stops["route_id"].astype(str) == route_id) - & (route_stops["shape_id"].astype(str) == shape_id) - ] - - # Fallback: match on route_id only (if shape_id is unknown or unmapped) - if filtered_stops.empty: - logger.info( - "No CSV rows for route_id=%r shape_id=%r — falling back to route_id only", - route_id, - shape_id, - ) - filtered_stops = route_stops[ - route_stops["route_id"].astype(str) == route_id - ] - - if filtered_stops.empty: - logger.warning( - "build_stop_time_updates: no stops for route_id=%r (CSV has routes=%s)", - route_id, - sorted(route_stops["route_id"].astype(str).unique().tolist()), - ) - return stop_time_update - - # Ensure ascending order so we walk stops in sequence - filtered_stops = filtered_stops.sort_values("stop_sequence") - - current_stop_sequence = _safe_int( - progression.get("current_stop_sequence"), default=-1 - ) - current_status = (progression.get("current_status") or "").upper() - - arrival_time = datetime.now() + timedelta( - minutes=random.randint(0, _ARRIVAL_MAX_MIN) - ) - - for _, row in filtered_stops.iterrows(): - stop_sequence = int(row["stop_sequence"]) - - # Skip stops the vehicle has already passed - if stop_sequence < current_stop_sequence: - continue - - # If the bus is currently stopped at this sequence, the next ETA is - # the following stop, so skip the current one. - if ( - current_status == "STOPPED_AT" - and stop_sequence == current_stop_sequence - ): - continue - - stop_entry = _generate_stop_entry( - arrival_time=arrival_time, - stop_sequence=stop_sequence, - stop_id=row["stop_id"], - uncertainty=_UNCERTAINTY_S, - ) - stop_time_update.append(stop_entry) - arrival_time += timedelta( - seconds=random.randint(_TIME_OFFSET_MIN_S, _TIME_OFFSET_MAX_S) - ) - - logger.debug( - "build_stop_time_updates: route_id=%s shape_id=%s current_seq=%s status=%s -> %d stops", - route_id, - shape_id, - current_stop_sequence, - current_status, - len(stop_time_update), - ) - return stop_time_update