From da3abfe0f8f1091f10ccb4b46102cdddf582ccc4 Mon Sep 17 00:00:00 2001 From: Mustafa Comoglu Date: Tue, 16 Jun 2026 20:26:12 +1000 Subject: [PATCH] [gui] Add built-in XYZ tile store (map.type = xyz) Add an XYZ / slippy-map tile store to the GUI map library so SeisComP GUIs can use basemaps from any standard XYZ server (OpenStreetMap, OpenTopoMap, ESRI, CartoDB, custom) out of the box, without an external plugin. - Implemented privately in an anonymous namespace in libs/seiscomp/gui/map/tilestores/xyz.cpp; no public header, so adding members later does not affect the library ABI. Registers itself with the TileStore factory as "xyz". - Features: per-zoom-level source bands (map.xyz.sources), single-source min/maxLevel, on-disk caching with TTL, negative-caching of HTTP>=400 tiles (map.xyz.missingTTL), {s} subdomain rotation, 256/512 tile size. - Link Qt Network into the gui library; document map.xyz.* in global_gui.xml. --- apps/gui/descriptions/global_gui.xml | 89 +++- libs/seiscomp/gui/CMakeLists.txt | 10 +- libs/seiscomp/gui/map/CMakeLists.txt | 1 + .../gui/map/tilestores/CMakeLists.txt | 5 + libs/seiscomp/gui/map/tilestores/xyz.cpp | 436 ++++++++++++++++++ 5 files changed, 534 insertions(+), 7 deletions(-) create mode 100644 libs/seiscomp/gui/map/tilestores/CMakeLists.txt create mode 100644 libs/seiscomp/gui/map/tilestores/xyz.cpp diff --git a/apps/gui/descriptions/global_gui.xml b/apps/gui/descriptions/global_gui.xml index 9c524c3ce..6652c3c49 100644 --- a/apps/gui/descriptions/global_gui.xml +++ b/apps/gui/descriptions/global_gui.xml @@ -91,11 +91,94 @@ - The tile store implementation provided by a plugin for - considering maps if a non-default tile store is configured - in 'map.location'. + The tile store implementation for considering maps if a + non-default tile store is configured in 'map.location'. + + The built-in 'xyz' store fetches tiles from any standard + XYZ / slippy-map server (OpenStreetMap, OpenTopoMap, ESRI, + CartoDB, custom servers); see the 'xyz' group below. + Additional stores may be provided by plugins. + + + Configuration of the built-in XYZ tile store + (map.type = xyz). The tile URL template in 'map.location' + uses the tokens {z} (zoom), {x} (column), {y} (row) and + {s} (subdomain), e.g. + https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png. + Note: some servers (e.g. ESRI) expect {z}/{y}/{x} order. + + + + Optional list of tile sources, each serving a contiguous + zoom-level band, to show different basemaps at different + zoom levels. Each entry has the form + "minLevel:maxLevel:urlTemplate"; only the first two + colons are separators, so the "https://" in the URL is + preserved. When set, this overrides 'map.location'. When + empty, 'map.location' is used with minLevel/maxLevel. + + + + + Minimum zoom level requested from the single-source + 'map.location'. Ignored when 'map.xyz.sources' is set. + + + + + Maximum zoom level requested from the single-source + 'map.location'. Ignored when 'map.xyz.sources' is set. + Provider maxima differ (OSM 19, Google ~21-22, ESRI 23); + the hard ceiling is 29. + + + + + Comma-separated subdomains substituted for {s} in the URL + template, rotated for basic load-balancing. Leave empty + if the server has no subdomain variants. + + + + + Edge length of square tiles in pixels. Must match what + the server actually serves; a mismatch renders blurry. + 256 = standard tiles, 512 = HiDPI/retina tiles. + + + + + Directory for caching downloaded tiles + (cacheDir/{z}/{x}/{y}). Leave empty to disable disk + caching. + + + + + How long a cached tile is considered fresh before it is + fetched again. -1 = never expire, 0 = disable caching. + + + + + How long to remember that a tile returned an HTTP error + (e.g. 404 above a server's max zoom) before retrying it, + to avoid re-requesting absent tiles on every redraw. + -1 = remember for the whole session, 0 = always retry. + + + + + HTTP User-Agent header sent with every tile request. Set + this to identify your institution when deploying + publicly; required by OSM's tile usage policy. + + + Allows to add custom layers that are included via plugins. diff --git a/libs/seiscomp/gui/CMakeLists.txt b/libs/seiscomp/gui/CMakeLists.txt index a8f532804..0aeb1ad70 100644 --- a/libs/seiscomp/gui/CMakeLists.txt +++ b/libs/seiscomp/gui/CMakeLists.txt @@ -19,15 +19,17 @@ SET(GUI_UI_HEADERS "") SC_LIB_INSTALL_HEADERS(GUI) IF (SC_GLOBAL_GUI_QT5) - FIND_PACKAGE(Qt5 COMPONENTS Svg Xml) + FIND_PACKAGE(Qt5 COMPONENTS Svg Xml Network) INCLUDE_DIRECTORIES(${Qt5Svg_INCLUDE_DIRS}) INCLUDE_DIRECTORIES(${Qt5Xml_INCLUDE_DIRS}) - SC_LIB_LINK_LIBRARIES(qt Qt5::Svg Qt5::Xml ${OPENSSL_LIBRARIES}) + INCLUDE_DIRECTORIES(${Qt5Network_INCLUDE_DIRS}) + SC_LIB_LINK_LIBRARIES(qt Qt5::Svg Qt5::Xml Qt5::Network ${OPENSSL_LIBRARIES}) ELSEIF (SC_GLOBAL_GUI_QT6) - FIND_PACKAGE(Qt6 REQUIRED COMPONENTS Svg Xml) + FIND_PACKAGE(Qt6 REQUIRED COMPONENTS Svg Xml Network) INCLUDE_DIRECTORIES(${Qt6Svg_INCLUDE_DIRS}) INCLUDE_DIRECTORIES(${Qt6Xml_INCLUDE_DIRS}) - SC_LIB_LINK_LIBRARIES(qt Qt6::Svg Qt6::Xml ${OPENSSL_LIBRARIES}) + INCLUDE_DIRECTORIES(${Qt6Network_INCLUDE_DIRS}) + SC_LIB_LINK_LIBRARIES(qt Qt6::Svg Qt6::Xml Qt6::Network ${OPENSSL_LIBRARIES}) ENDIF() SC_LIB_VERSION(qt ${SC_COMMON_VERSION} ${SC_COMMON_VERSION_MAJOR}) diff --git a/libs/seiscomp/gui/map/CMakeLists.txt b/libs/seiscomp/gui/map/CMakeLists.txt index 72292d371..9d5349c8c 100644 --- a/libs/seiscomp/gui/map/CMakeLists.txt +++ b/libs/seiscomp/gui/map/CMakeLists.txt @@ -31,4 +31,5 @@ SET(GUI_MAP_MOC_HEADERS SC_ADD_GUI_SUBDIR_SOURCES(GUI_MAP projections) SC_ADD_GUI_SUBDIR_SOURCES(GUI_MAP layers) +SC_ADD_GUI_SUBDIR_SOURCES(GUI_MAP tilestores) SC_SETUP_GUI_LIB_SUBDIR(GUI_MAP) diff --git a/libs/seiscomp/gui/map/tilestores/CMakeLists.txt b/libs/seiscomp/gui/map/tilestores/CMakeLists.txt new file mode 100644 index 000000000..a651e0e06 --- /dev/null +++ b/libs/seiscomp/gui/map/tilestores/CMakeLists.txt @@ -0,0 +1,5 @@ +SET(GUI_MAP_TILESTORES_SOURCES + xyz.cpp +) + +SC_SETUP_GUI_LIB_SUBDIR(GUI_MAP_TILESTORES) diff --git a/libs/seiscomp/gui/map/tilestores/xyz.cpp b/libs/seiscomp/gui/map/tilestores/xyz.cpp new file mode 100644 index 000000000..0e06db137 --- /dev/null +++ b/libs/seiscomp/gui/map/tilestores/xyz.cpp @@ -0,0 +1,436 @@ +/*************************************************************************** + * Copyright (C) gempa GmbH * + * All rights reserved. * + * Contact: gempa GmbH (seiscomp-dev@gempa.de) * + * * + * GNU Affero General Public License Usage * + * This file may be used under the terms of the GNU Affero * + * Public License version 3.0 as published by the Free Software Foundation * + * and appearing in the file LICENSE included in the packaging of this * + * file. Please review the following information to ensure the GNU Affero * + * Public License version 3.0 requirements will be met: * + * https://www.gnu.org/licenses/agpl-3.0.html. * + * * + * Other Usage * + * Alternatively, this file may be used in accordance with the terms and * + * conditions contained in a signed written agreement between you and * + * gempa GmbH. * + ***************************************************************************/ + + +#define SEISCOMP_COMPONENT Gui::XYZTileStore + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace Seiscomp { +namespace Gui { +namespace Map { +namespace { + + +// Tile store of type "xyz": fetches map tiles from any standard XYZ / +// slippy-map server (OpenStreetMap, OpenTopoMap, ESRI, CartoDB, custom +// servers) over HTTP and caches them on disk. +// +// The implementation lives entirely in this private namespace; it is not part +// of the public API. Adding members here therefore never affects the library +// ABI. It registers itself with the TileStore factory under the name "xyz", +// selected via map.type = xyz. +// +// It is a QObject (without Q_OBJECT/moc) only so that it can own the network +// access manager and act as the connection context; replies are dispatched +// through a member function via the functor-based connect overload. +class XYZTileStore : public QObject, public TileStore { + public: + XYZTileStore() = default; + ~XYZTileStore() override { + refresh(); + } + + public: + bool open(MapsDesc &desc) override { + QString defaultURL = desc.location.trimmed(); + + auto *app = Client::Application::Instance(); + + // Single-source band, also the default for sources entries that + // omit a band. + int defaultMin = 0; + int defaultMax = 19; + if ( app ) { + try { defaultMin = app->configGetInt("map.xyz.minLevel"); } catch ( ... ) {} + try { defaultMax = app->configGetInt("map.xyz.maxLevel"); } catch ( ... ) {} + } + + // "map.xyz.sources": one entry per zoom band, + // "minLevel:maxLevel:urlTemplate". Only the first two colons are + // separators, so the "https://" in the URL stays intact. + if ( app ) { + try { + for ( const auto &raw : app->configGetStrings("map.xyz.sources") ) { + QString entry = QString::fromStdString(raw).trimmed(); + if ( entry.isEmpty() ) + continue; + + int c1 = entry.indexOf(':'); + int c2 = c1 >= 0 ? entry.indexOf(':', c1 + 1) : -1; + if ( c1 < 0 || c2 < 0 ) { + SEISCOMP_WARNING("xyz: ignoring malformed map.xyz.sources entry " + "(want minLevel:maxLevel:url): %s", + qUtf8Printable(entry)); + continue; + } + + bool okMin = false, okMax = false; + Source src; + src.minLevel = entry.left(c1).trimmed().toInt(&okMin); + src.maxLevel = entry.mid(c1 + 1, c2 - c1 - 1).trimmed().toInt(&okMax); + src.url = entry.mid(c2 + 1).trimmed(); + if ( !okMin || !okMax || src.url.isEmpty() || src.minLevel > src.maxLevel ) { + SEISCOMP_WARNING("xyz: ignoring invalid map.xyz.sources entry: %s", + qUtf8Printable(entry)); + continue; + } + _sources.push_back(src); + } + } + catch ( ... ) {} + } + + // No per-source bands configured: fall back to map.location. + if ( _sources.isEmpty() ) { + if ( defaultURL.isEmpty() ) { + SEISCOMP_ERROR("xyz: no tile source - set map.location to an XYZ URL " + "template (e.g. https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png) " + "or provide map.xyz.sources entries"); + return false; + } + _sources.push_back(Source{defaultMin, defaultMax, defaultURL}); + } + + // Overall level bounds = union across sources. + _minLevel = _sources.front().minLevel; + _maxLevel = _sources.front().maxLevel; + for ( const auto &s : _sources ) { + _minLevel = std::min(_minLevel, s.minLevel); + _maxLevel = std::max(_maxLevel, s.maxLevel); + } + if ( _maxLevel > static_cast(TileIndex::MaxLevel) ) + _maxLevel = static_cast(TileIndex::MaxLevel); + + if ( app ) { + try { _cacheDuration = app->configGetInt("map.xyz.cacheDuration"); } catch ( ... ) {} + try { _missingTTL = app->configGetInt("map.xyz.missingTTL"); } catch ( ... ) {} + try { + _cacheDir = QString::fromStdString(app->configGetString("map.xyz.cacheDir")); + } + catch ( ... ) {} + try { + _userAgent = QString::fromStdString(app->configGetString("map.xyz.userAgent")); + } + catch ( ... ) {} + try { + QString s = QString::fromStdString(app->configGetString("map.xyz.subdomains")); + for ( auto &sub : s.split(',', Qt::SkipEmptyParts) ) + _subdomains << sub.trimmed(); + } + catch ( ... ) {} + try { + // Stored as a string so scconfig presents a 256/512 dropdown. + bool ok = false; + int sz = QString::fromStdString(app->configGetString("map.xyz.tileSize")) + .trimmed().toInt(&ok); + if ( ok && sz > 0 ) + _tilesize = QSize(sz, sz); + else + SEISCOMP_WARNING("xyz: ignoring invalid map.xyz.tileSize"); + } + catch ( ... ) {} + } + + if ( _tilesize.isEmpty() ) + _tilesize = QSize(256, 256); + + bool needSubdomains = false; + for ( const auto &s : _sources ) + needSubdomains = needSubdomains || s.url.contains("{s}"); + if ( _subdomains.isEmpty() && needSubdomains ) + _subdomains << "a" << "b" << "c"; + + _projection = Mercator; + + _nam = new QNetworkAccessManager(this); + connect(_nam, &QNetworkAccessManager::finished, + this, &XYZTileStore::onRequestFinished); + + if ( !_cacheDir.isEmpty() ) + QDir().mkpath(_cacheDir); + + SEISCOMP_INFO("xyz: tile store opened - %d source(s) levels=%d..%d cache=%s ttl=%ds", + static_cast(_sources.size()), _minLevel, _maxLevel, + _cacheDir.isEmpty() ? "disabled" : qUtf8Printable(_cacheDir), + _cacheDuration); + for ( const auto &s : _sources ) + SEISCOMP_INFO("xyz: level %2d..%-2d -> %s", + s.minLevel, s.maxLevel, qUtf8Printable(s.url)); + return true; + } + + int maxLevel() const override { + return _maxLevel; + } + + LoadResult load(QImage &img, const TileIndex &tile) override { + if ( _inflight.contains(tile.id) ) + return Deferred; + + // Negative cache: don't re-hammer the server for tiles it already + // told us it doesn't have (HTTP >= 400). Respect map.xyz.missingTTL. + auto miss = _missing.constFind(tile.id); + if ( miss != _missing.constEnd() ) { + if ( _missingTTL < 0 || + miss.value() + _missingTTL > QDateTime::currentSecsSinceEpoch() ) + return Error; + _missing.erase(_missing.find(tile.id)); + } + + if ( !_cacheDir.isEmpty() ) { + QString path = cachePath(tile); + if ( QFile::exists(path) && isCacheFresh(path) ) { + if ( img.load(path) ) + return OK; + QFile::remove(path); + } + } + + startRequest(tile); + return Deferred; + } + + QString getID(const TileIndex &tile) const override { + return QString("%1/%2/%3") + .arg(tile.level()).arg(tile.column()).arg(tile.row()); + } + + bool validate(int level, int column, int row) const override { + if ( level < _minLevel || level > _maxLevel ) + return false; + const int n = 1 << level; + return column >= 0 && column < n && row >= 0 && row < n; + } + + bool hasPendingRequests() const override { + return !_inflight.isEmpty(); + } + + void refresh() override { + for ( auto *reply : _replyMap.keys() ) + reply->abort(); + _replyMap.clear(); + _inflight.clear(); + } + + private: + using TileId = TileIndex::Storage; + + //! A tile source serving a contiguous zoom-level band. Each source + //! carries its own maxLevel because providers differ: OSM standard + //! caps at 19, Google ~21-22, ESRI ArcGIS up to 23. The hard ceiling + //! is TileIndex::MaxLevel. + struct Source { + int minLevel{0}; + int maxLevel{19}; + QString url; + }; + + const Source *sourceForLevel(int level) const { + for ( const auto &s : _sources ) { + if ( level >= s.minLevel && level <= s.maxLevel ) + return &s; + } + // Gap between bands: fall back to the source with the nearest edge + // so a tile still renders (scaled) rather than showing a hole. + const Source *best = nullptr; + int bestDist = 0; + for ( const auto &s : _sources ) { + int d = std::min(std::abs(level - s.minLevel), std::abs(level - s.maxLevel)); + if ( !best || d < bestDist ) { + best = &s; + bestDist = d; + } + } + return best; + } + + QString buildURL(const TileIndex &tile) { + const Source *src = sourceForLevel(tile.level()); + if ( !src ) + return QString(); + + QString url = src->url; + url.replace("{z}", QString::number(tile.level())); + url.replace("{x}", QString::number(tile.column())); + url.replace("{y}", QString::number(tile.row())); + + if ( !_subdomains.isEmpty() ) { + url.replace("{s}", _subdomains[_subdomainIndex % _subdomains.size()]); + ++_subdomainIndex; + } + + return url; + } + + QString cachePath(const TileIndex &tile) const { + return QString("%1/%2/%3/%4") + .arg(_cacheDir) + .arg(tile.level()) + .arg(tile.column()) + .arg(tile.row()); + } + + bool isCacheFresh(const QString &path) const { + if ( _cacheDuration < 0 ) return true; + if ( _cacheDuration == 0 ) return false; + return QFileInfo(path).lastModified().secsTo(QDateTime::currentDateTime()) + < _cacheDuration; + } + + void startRequest(const TileIndex &tile) { + _inflight.insert(tile.id); + + QNetworkRequest req{QUrl{buildURL(tile)}}; + req.setHeader(QNetworkRequest::UserAgentHeader, _userAgent); + req.setAttribute(QNetworkRequest::HttpPipeliningAllowedAttribute, true); + req.setAttribute(QNetworkRequest::Http2AllowedAttribute, true); + req.setAttribute(QNetworkRequest::RedirectPolicyAttribute, + QNetworkRequest::NoLessSafeRedirectPolicy); + + QNetworkReply *reply = _nam->get(req); + _replyMap.insert(reply, tile.id); + } + + void onRequestFinished(QNetworkReply *reply) { + reply->deleteLater(); + + auto it = _replyMap.find(reply); + if ( it == _replyMap.end() ) + return; + + TileId tileId = it.value(); + _replyMap.erase(it); + _inflight.remove(tileId); + + TileIndex tile; + tile.id = tileId; + + if ( reply->error() != QNetworkReply::NoError ) { + if ( reply->error() != QNetworkReply::OperationCanceledError ) { + SEISCOMP_WARNING("xyz: fetch error for %s: %s", + qUtf8Printable(getID(tile)), + qUtf8Printable(reply->errorString())); + } + loadingCancelled(tile); + return; + } + + int httpStatus = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if ( httpStatus >= 400 ) { + SEISCOMP_WARNING("xyz: HTTP %d for tile %s", + httpStatus, qUtf8Printable(getID(tile))); + if ( _missingTTL != 0 ) + _missing.insert(tileId, QDateTime::currentSecsSinceEpoch()); + loadingCancelled(tile); + return; + } + + QByteArray data = reply->readAll(); + if ( data.isEmpty() ) { + loadingCancelled(tile); + return; + } + + QImage img; + if ( !img.loadFromData(data) ) { + SEISCOMP_WARNING("xyz: image decode failed for tile %s", + qUtf8Printable(getID(tile))); + loadingCancelled(tile); + return; + } + + // One-time sanity check: the configured map.xyz.tileSize must match + // the pixels the server actually serves, otherwise the projection + // scale (which uses tileSize) picks the wrong level and the map + // looks blurry. + if ( !_tileSizeChecked ) { + _tileSizeChecked = true; + if ( img.size() != _tilesize ) + SEISCOMP_WARNING("xyz: server tile is %dx%d but map.xyz.tileSize is %dx%d - " + "set map.xyz.tileSize to %d to avoid blurry rendering", + img.width(), img.height(), + _tilesize.width(), _tilesize.height(), img.width()); + } + + // Only persist when caching is actually enabled for reads + // (cacheDuration == 0 disables the cache entirely). + if ( !_cacheDir.isEmpty() && _cacheDuration != 0 ) { + QString path = cachePath(tile); + QDir().mkpath(QFileInfo(path).absolutePath()); + QFile f(path); + if ( f.open(QIODevice::WriteOnly) ) + f.write(data); + } + + loadingComplete(img, tile); + } + + private: + QNetworkAccessManager *_nam{nullptr}; + QVector _sources; + QStringList _subdomains; + int _subdomainIndex{0}; + int _minLevel{0}; + int _maxLevel{19}; + QString _cacheDir; + int _cacheDuration{86400}; + QString _userAgent{"SeisComP-xyztiles/1.0"}; + + int _missingTTL{300}; + QHash _missing; + + bool _tileSizeChecked{false}; + + QHash _replyMap; + QSet _inflight; +}; + + +REGISTER_TILESTORE_INTERFACE(XYZTileStore, "xyz"); + + +} +} +} +}