From 4154d6c4ca7585f6fec2fae2b4ccf29f45b9028a Mon Sep 17 00:00:00 2001 From: Ryan Dougherty Date: Fri, 31 Jul 2026 11:31:21 -0700 Subject: [PATCH 1/4] Transition deprecated Scattermapbox plots to more general Scattermap plots. --- .../fusion_engine_client/analysis/analyzer.py | 90 ++++++++++++------- 1 file changed, 60 insertions(+), 30 deletions(-) diff --git a/python/fusion_engine_client/analysis/analyzer.py b/python/fusion_engine_client/analysis/analyzer.py index ead1a9fd..f4060941 100755 --- a/python/fusion_engine_client/analysis/analyzer.py +++ b/python/fusion_engine_client/analysis/analyzer.py @@ -88,6 +88,37 @@ def _data_to_table(col_titles: List[str], values: List[List[Any]], row_major: bo return table_html.replace('\n', '') +def _build_map_style(mapbox_token: Optional[str]): + """! + @brief Build a `layout.map.style` value for a MapLibre-based Scattermap figure. + + If a Mapbox access token is available, pull Mapbox satellite tiles via a custom raster style spec (the mechanism + MapLibre-based maps use in place of the old `layout.mapbox.accesstoken` field, which no longer exists). Otherwise, + fall back to Plotly's built-in token-free `satellite-streets` style, which serves ESRI World Imagery aerial tiles + (max zoom 16, lower resolution than Mapbox) with OpenMapTiles street labels drawn on top. + """ + if not mapbox_token: + return 'satellite-streets' + + return { + 'version': 8, + 'sources': { + 'mapbox-satellite': { + 'type': 'raster', + 'tiles': [ + f'https://api.mapbox.com/v4/mapbox.satellite/{{z}}/{{x}}/{{y}}@2x.jpg90' + f'?access_token={mapbox_token}' + ], + 'tileSize': 256, + 'attribution': '© Mapbox', + }, + }, + 'layers': [ + {'id': 'mapbox-satellite-layer', 'type': 'raster', 'source': 'mapbox-satellite'}, + ], + } + + _page_template = '''\ @@ -1159,9 +1190,9 @@ def plot_map(self, mapbox_token, reference: Optional[ReferenceData] = None): mapbox_token = self.get_mapbox_token(mapbox_token) if mapbox_token is None or mapbox_token == "": self.logger.info('*' * 80 + '\n\n' + - 'Mapbox token not specified. Disabling satellite imagery. For satellite imagery,\n' - 'please provide a Mapbox token using --mapbox-token or by setting the\n' - 'MAPBOX_ACCESS_TOKEN environment variable.' + + 'Mapbox token not specified. Falling back to lower-resolution free satellite\n' + 'imagery. For high-resolution imagery, please provide a Mapbox token using\n' + '--mapbox-token or by setting the MAPBOX_ACCESS_TOKEN environment variable.' + '\n\n' + '*' * 80) self._mapbox_token_missing = True mapbox_token = None @@ -1244,10 +1275,10 @@ def _plot_data(name, selected_idx, flags, source_id, lla_deg, customdata_all, ma if np.any(is_nav_engine): idx = is_nav_engine - map_data.append(go.Scattermapbox(lat=lla_deg[0, idx], lon=lla_deg[1, idx], name=name, - customdata=[customdata_all[i] for i in np.nonzero(idx)[0]], - hovertemplate=_with_bold_name(hovertemplate, name), - legendgroup=legendgroup, visible=visible, **style)) + map_data.append(go.Scattermap(lat=lla_deg[0, idx], lon=lla_deg[1, idx], name=name, + customdata=[customdata_all[i] for i in np.nonzero(idx)[0]], + hovertemplate=_with_bold_name(hovertemplate, name), + legendgroup=legendgroup, visible=visible, **style)) indices_by_engine['Nav Engine'].append(len(map_data) - 1) if np.any(is_gnss_rx): @@ -1255,17 +1286,17 @@ def _plot_data(name, selected_idx, flags, source_id, lla_deg, customdata_all, ma style['marker']['opacity'] = 0.5 style['marker']['size'] = 5 gnss_name = name + ' (Receiver Solution)' - map_data.append(go.Scattermapbox(lat=lla_deg[0, idx], lon=lla_deg[1, idx], - name=gnss_name, - customdata=[customdata_all[i] for i in np.nonzero(idx)[0]], - hovertemplate=_with_bold_name(hovertemplate, gnss_name), - legendgroup=legendgroup, visible=visible, **style)) + map_data.append(go.Scattermap(lat=lla_deg[0, idx], lon=lla_deg[1, idx], + name=gnss_name, + customdata=[customdata_all[i] for i in np.nonzero(idx)[0]], + hovertemplate=_with_bold_name(hovertemplate, gnss_name), + legendgroup=legendgroup, visible=visible, **style)) indices_by_engine['Receiver Solution'].append(len(map_data) - 1) else: # If there's no data, draw a dummy trace so it shows up in the legend anyway. - map_data.append(go.Scattermapbox(lat=[np.nan], lon=[np.nan], name=name, legendgroup=legendgroup, - visible='legendonly', **style)) + map_data.append(go.Scattermap(lat=[np.nan], lon=[np.nan], name=name, legendgroup=legendgroup, + visible='legendonly', **style)) indices_by_engine['Nav Engine'].append(len(map_data) - 1) # Read the pose data. @@ -1340,7 +1371,7 @@ def _plot_data(name, selected_idx, flags, source_id, lla_deg, customdata_all, ma # Add reference/truth data to the map, if available, restricted to the time range covered by the pose data. # Built as a separate list and prepended below (rather than appended to map_data directly) so the reference - # is drawn first -- Scattermapbox layers later traces on top, and we want the pose data on top of the + # is drawn first -- Scattermap layers later traces on top, and we want the pose data on top of the # reference, not the other way around. ref_traces = [] if reference is not None and (reference.is_stationary or overall_gps_t_min is not None): @@ -1381,12 +1412,12 @@ def _plot_data(name, selected_idx, flags, source_id, lla_deg, customdata_all, ma trace_customdata = None else: trace_customdata = [ref_customdata[i] for i in np.nonzero(idx)[0]] - ref_traces.append(go.Scattermapbox(lat=ref_lla_deg[0, idx], lon=ref_lla_deg[1, idx], - name=name, mode='markers', - marker={'size': 8, 'color': color}, - showlegend=True, legendgroup='ref', - customdata=trace_customdata, - hovertemplate=_with_bold_name(ref_hovertemplate, name))) + ref_traces.append(go.Scattermap(lat=ref_lla_deg[0, idx], lon=ref_lla_deg[1, idx], + name=name, mode='markers', + marker={'size': 8, 'color': color}, + showlegend=True, legendgroup='ref', + customdata=trace_customdata, + hovertemplate=_with_bold_name(ref_hovertemplate, name))) if ref_traces: # Shift the pose traces' button indices to account for the reference traces now being inserted ahead of @@ -1400,8 +1431,8 @@ def _plot_data(name, selected_idx, flags, source_id, lla_deg, customdata_all, ma # Create the map. title = 'Vehicle Trajectory' if mapbox_token is None: - title += '
For satellite imagery, please provide a Mapbox token using --mapbox-token or by setting ' \ - 'MAPBOX_ACCESS_TOKEN.' + title += '
For higher-resolution satellite imagery, please provide a Mapbox token using ' \ + '--mapbox-token or by setting MAPBOX_ACCESS_TOKEN.' layout = go.Layout( autosize=True, @@ -1412,8 +1443,7 @@ def _plot_data(name, selected_idx, flags, source_id, lla_deg, customdata_all, ma title=dict(text=title, x=0, xanchor='left', xref='paper'), # Reduce padding around the map, leaving enough space for the title. margin=dict(l=16, r=16, t=70, b=8), - mapbox=dict( - accesstoken=mapbox_token, + map=dict( bearing=0, center=dict( lat=lla_deg[0, 0], @@ -1421,7 +1451,7 @@ def _plot_data(name, selected_idx, flags, source_id, lla_deg, customdata_all, ma ), pitch=0, zoom=18, - style='open-street-map' if mapbox_token is None else 'satellite-streets', + style=_build_map_style(mapbox_token), ), ) @@ -1464,7 +1494,7 @@ def _plot_data(name, selected_idx, flags, source_id, lla_deg, customdata_all, ma # shrink after the time scale renders. # # The map itself starts hidden (`visibility:hidden`, which still reserves its final layout space, unlike - # `display:none`) -- even with the container correctly sized up front, Plotly's own WebGL/mapbox-gl + # `display:none`) -- even with the container correctly sized up front, Plotly's own WebGL/MapLibre # rendering doesn't necessarily catch up to a resize() call within the same paint, so revealing it right # away can still show one frame at the wrong (window-sized) dimensions overlapping the slider. It's # revealed by JS (plotly_map_time_slider.js) once Plotly itself reports the post-resize redraw is done. @@ -3418,9 +3448,9 @@ def generate_index(self, reference: Optional[ReferenceData] = None, auto_open: b if self._mapbox_token_missing: self.summary += """\n

- Warning: Mapbox token not specified. Generated map using Open Street Maps - street data. For satellite imagery, please request a free access token from - https://account.mapbox.com/access-tokens, then provide the token by + Warning: Mapbox token not specified. Generated map using free, lower-resolution + satellite imagery. For high-resolution imagery, please request a free access + token from https://account.mapbox.com/access-tokens, then provide the token by specifying --mapbox-token or setting the MAPBOX_ACCESS_TOKEN environment variable.

From 94d02fca9e0b7e9ff650aa730731ec7ebb18d7b0 Mon Sep 17 00:00:00 2001 From: Ryan Dougherty Date: Fri, 31 Jul 2026 13:17:30 -0700 Subject: [PATCH 2/4] Simplify start, stop time index case handling. --- .../parsers/file_index.py | 51 ++++++++++--------- python/tests/test_file_index.py | 13 +++++ 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/python/fusion_engine_client/parsers/file_index.py b/python/fusion_engine_client/parsers/file_index.py index 08bdcaa2..32e52e20 100644 --- a/python/fusion_engine_client/parsers/file_index.py +++ b/python/fusion_engine_client/parsers/file_index.py @@ -384,35 +384,40 @@ def get_time_range(self, start: Union[Timestamp, float] = None, stop: Union[Time raise IndexError(f'No P1 timestamps present in index. Cannot apply time bounds. ' f'[start={start}, stop={stop}]') else: - # Note: The index stores only the integer part of the timestamp. - - # If self._data['time'] ends _before_ `start``, use 0 as start_idx. If self._data['time'] ends _after_ - # `end`, use len(self._data['time']) as end_idx. - with np.errstate(invalid='ignore'): - start_idx = find_first(self._data['time'] >= np.floor(start)) if start is not None else 0 - end_idx = find_first(self._data['time'] >= stop) if stop is not None else len(self._data) - - # Corner case: if all messages with timestamps are >= stop (i.e., the log starts after the stop time), if - # there are some messages at the start of the log that do not have timestamps, find_first() will include - # them but we don't want that. For example, if stop is 4 and we have: - # {nan, 6, 7, 8} - # we expect end_idx = -1 (i.e., nothing in range), not end_idx = 1 (i.e., include the nan message). - nan_idx = np.isnan(self._data['time']) - if end_idx >= 1 and np.all(nan_idx[:end_idx]): - end_idx = -1 - - # Note: start_idx or end_idx == -1 indicates there was no data in the time range. + # Both bounds are intentionally over-inclusive: the index stores only the integer part of each timestamp, so + # floor `start` to avoid dropping a message whose true time is in range. `stop` needs no adjustment. + # + # nan entries (no P1Time) never satisfy `>=`, so find_first() either lands on a timestamped entry or returns + # -1. Treat -1 as len(self._data): an empty range for `start`, or the end of the data for `stop`. + def _first_at_or_after(time_sec: float) -> int: + with np.errstate(invalid='ignore'): + idx = find_first(self._data['time'] >= time_sec) + return len(self._data) if idx < 0 else idx + + # Messages without P1Time are in range while the surrounding P1 time is in range, so an omitted bound takes + # in the initial/final block of them at that end of the log. Keep this consistent with @ref + # TimeRange.is_in_range(), which is used to read a log with no index. + start_idx = _first_at_or_after(np.floor(start)) if start is not None else 0 + end_idx = _first_at_or_after(stop) if stop is not None else len(self._data) + + # With `start` omitted the range begins at index 0, which may cover an initial block of messages without + # P1Time. If it covers _nothing else_, no timestamped data is in range, so the range is empty: stop=4 over + # {nan, nan, 6, 7, 8} must yield nothing, not end_idx == 2. Note that is_in_range() would return the two + # leading messages; this is our one intentional deviation. + is_nan = np.isnan(self._data['time']) + if stop is not None and np.all(is_nan[start_idx:end_idx]): + end_idx = start_idx + idx = np.full_like(self._data['time'], False, dtype=bool) - if start_idx >= 0 and end_idx >= 0: - idx[start_idx:end_idx] = True + idx[start_idx:end_idx] = True if hint in ('all_nans', 'remove_nans'): if hint == 'all_nans': - idx[nan_idx] = True + idx[is_nan] = True elif hint == 'remove_nans': - idx[nan_idx] = False + idx[is_nan] = False elif hint != 'include_nans': - raise ValueError('Unrecognized control hint.') + raise ValueError(f'Unrecognized control hint: "{hint}".') return FileIndex(data=self._data[idx], t0=self.t0) diff --git a/python/tests/test_file_index.py b/python/tests/test_file_index.py index ab6b1bd3..bc6048fb 100644 --- a/python/tests/test_file_index.py +++ b/python/tests/test_file_index.py @@ -194,6 +194,19 @@ def _lower_bound(time): assert (sliced_index.offset == [e[2] for e in raw]).all() assert (sliced_index.message_index == [e[3] for e in raw]).all() + # End time beyond end of data: the range extends to the end of the data. + sliced_index = index[TimeRange(end=1000.0, absolute=True)] + assert _test_time(sliced_index.time, RAW_DATA) + assert (sliced_index.offset == [e[2] for e in RAW_DATA]).all() + assert (sliced_index.message_index == [e[3] for e in RAW_DATA]).all() + + # Start time within the data, end time beyond the end of the data. + sliced_index = index[TimeRange(start=3.0, end=1000.0, absolute=True)] + raw = RAW_DATA[_lower_bound(3.0):] + assert _test_time(sliced_index.time, raw) + assert (sliced_index.offset == [e[2] for e in raw]).all() + assert (sliced_index.message_index == [e[3] for e in raw]).all() + # Start time beyond end of data. sliced_index = index[TimeRange(start=1000.0, absolute=True)] assert len(sliced_index) == 0 From 289033d9d6fa9f58bda3aa6c7909c6a89d7fa68b Mon Sep 17 00:00:00 2001 From: Ryan Dougherty Date: Fri, 31 Jul 2026 15:46:33 -0700 Subject: [PATCH 3/4] Ignore *venv/ as well. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c1bb3eff..33b33b75 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ build/ # Python files. venv*/ +*venv/ *.pyc # IDE project settings. From 3e660a4a7176748efbc33065f5e0927f137174d0 Mon Sep 17 00:00:00 2001 From: Ryan Dougherty Date: Fri, 31 Jul 2026 15:47:49 -0700 Subject: [PATCH 4/4] Extend file index test coverage. --- python/tests/test_file_index.py | 113 ++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/python/tests/test_file_index.py b/python/tests/test_file_index.py index bc6048fb..4c7782c7 100644 --- a/python/tests/test_file_index.py +++ b/python/tests/test_file_index.py @@ -19,6 +19,10 @@ RAW_DATA = [(*entry, i) for i, entry in enumerate(RAW_DATA)] +def _make_index(entries): + return FileIndex(data=[(*entry, i) for i, entry in enumerate(entries)]) + + def _test_time(time, raw_data): raw_time = [e[0] for e in raw_data] raw_is_none = [e is None for e in raw_time] @@ -230,6 +234,115 @@ def _lower_bound(time): assert (sliced_index.message_index == [e[3] for e in raw]).all() +def test_time_range_slice_corner_cases(): + index = FileIndex(data=RAW_DATA) + + # The index stores only the integer part of each timestamp, so both bounds are intentionally over-inclusive: a + # fractional `start` is floored so we don't drop a message whose true time may fall in range, and a fractional + # `stop` keeps the messages sharing its integer second. + sliced_index = index[TimeRange(start=2.5, end=3.5, absolute=True)] + raw = RAW_DATA[2:6] + assert _test_time(sliced_index.time, raw) + assert (sliced_index.offset == [e[2] for e in raw]).all() + assert (sliced_index.message_index == [e[3] for e in raw]).all() + + # Empty and inverted ranges. + assert len(index[TimeRange(start=3.0, end=3.0, absolute=True)]) == 0 + assert len(index[TimeRange(start=3.0, end=2.0, absolute=True)]) == 0 + + # An initial block of messages without P1Time is treated as being "before" a specified `start`, and is included only + # when `start` is not specified. Note that TimeRange normalizes an absolute start of 0.0 to None ("not specified"), + # so use a nonzero start that still precedes the first timestamp. + leading_index = _make_index([ + (None, MessageType.VERSION_INFO, 0), + (None, MessageType.VERSION_INFO, 10), + (Timestamp(6.0), MessageType.POSE, 20), + (Timestamp(7.0), MessageType.POSE, 30), + (Timestamp(8.0), MessageType.POSE, 40), + ]) + sliced_index = leading_index[TimeRange(end=7.0, absolute=True)] + assert (sliced_index.message_index == [0, 1, 2]).all() + sliced_index = leading_index[TimeRange(start=1.0, end=7.0, absolute=True)] + assert (sliced_index.message_index == [2]).all() + + # Corner case: the log starts after the requested stop time, but begins with messages without P1Time. Those + # messages must not be treated as in range, since no timestamped data is in range at all. + assert len(leading_index[TimeRange(end=4.0, absolute=True)]) == 0 + + # The leading messages are still returned if the caller explicitly asks for all nans. + sliced_index = leading_index[TimeRange(end=4.0, absolute=True), 'all_nans'] + assert (sliced_index.message_index == [0, 1]).all() + + # Same corner case, but with a `stop` of 0.0: the range is empty, so the leading messages are still out of range. + # Note that this requires testing `stop is not None`, not the truthiness of `stop`. + assert len(leading_index[TimeRange(end=0.0, absolute=True)]) == 0 + assert len(leading_index.get_time_range(stop=0.0)) == 0 + + # A final block of messages without P1Time is in range as long as the range has not already ended, i.e. as long as + # no timestamp >= `stop` was found. This matches TimeRange.is_in_range(), which only ends a range on a timestamped + # message, so that an indexed read and a non-indexed read of the same log agree. + trailing_index = _make_index([ + (Timestamp(1.0), MessageType.POSE, 0), + (Timestamp(2.0), MessageType.POSE, 10), + (None, MessageType.VERSION_INFO, 20), + (None, MessageType.VERSION_INFO, 30), + ]) + sliced_index = trailing_index[TimeRange(start=1.0, absolute=True)] + assert (sliced_index.message_index == [0, 1, 2, 3]).all() + sliced_index = trailing_index[TimeRange(start=1.0, end=1000.0, absolute=True)] + assert (sliced_index.message_index == [0, 1, 2, 3]).all() + + # ...but once a timestamp >= `stop` is found, the range ends there, so a following block is out of range. + sliced_index = trailing_index[TimeRange(start=1.0, end=2.0, absolute=True)] + assert (sliced_index.message_index == [0]).all() + + # A block of messages without P1Time between two timestamps is in range as usual. + interior_index = _make_index([ + (Timestamp(1.0), MessageType.POSE, 0), + (None, MessageType.VERSION_INFO, 10), + (Timestamp(9.0), MessageType.POSE, 20), + ]) + sliced_index = interior_index[TimeRange(start=1.0, end=1000.0, absolute=True)] + assert (sliced_index.message_index == [0, 1, 2]).all() + sliced_index = interior_index[TimeRange(start=1.0, end=9.0, absolute=True)] + assert (sliced_index.message_index == [0, 1]).all() + + # Untimed messages within the range may be dropped on request. + sliced_index = index.get_time_range(time_range=TimeRange(start=1.0, end=3.0, absolute=True), hint='remove_nans') + raw = [e for e in RAW_DATA[1:5] if e[0] is not None] + assert _test_time(sliced_index.time, raw) + assert (sliced_index.message_index == [e[3] for e in raw]).all() + + with pytest.raises(ValueError): + index.get_time_range(time_range=TimeRange(start=1.0, absolute=True), hint='not_a_hint') + + # No time bounds specified: return everything. + sliced_index = index.get_time_range() + assert (sliced_index.message_index == [e[3] for e in RAW_DATA]).all() + + # An index containing _only_ messages without P1Time, but which still knows its t0 -- as produced by an 'all_nans' + # slice -- can be sliced by time again. Its handling depends on which bounds were specified: + nan_only = index[TimeRange(start=1000.0, absolute=True), 'all_nans'] + assert len(nan_only) > 0 and np.isnan(nan_only.time).all() and nan_only.t0 is not None + + # - Neither bound: it is okay to return messages outside the extreme P1Times, so return everything. + assert (nan_only.get_time_range().message_index == list(nan_only.message_index)).all() + # - `stop` specified: no timestamped data is in range, so the range is empty. + assert len(nan_only[TimeRange(end=1000.0, absolute=True)]) == 0 + assert len(nan_only[TimeRange(start=1.0, end=1000.0, absolute=True)]) == 0 + # - Only `start` specified: there is no timestamp at or after it, so the range is empty as well. + assert len(nan_only[TimeRange(start=1.0, absolute=True)]) == 0 + + # An empty index has nothing to bound. Note that __getitem__() short-circuits empty indices, so call + # get_time_range() directly to exercise its own empty handling. + assert len(FileIndex().get_time_range(time_range=TimeRange(start=1.0, end=2.0, absolute=True))) == 0 + assert len(FileIndex()[TimeRange(start=1.0, end=2.0, absolute=True)]) == 0 + + # `start`/`stop` and a TimeRange are mutually exclusive. + with pytest.raises(ValueError): + index.get_time_range(start=1.0, time_range=TimeRange(start=1.0, absolute=True)) + + def test_time_slice_no_p1_time(): def _lower_bound(time): return next(i for i, e in enumerate(RAW_DATA) if (e[0] is not None and e[0] >= time))