Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: OpenGIS Simple Features Reference Implementation
4 : * Purpose: Implementation of PMTiles
5 : * Author: Even Rouault <even.rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2023, Planet Labs
9 : *
10 : * SPDX-License-Identifier: MIT
11 : ****************************************************************************/
12 :
13 : #ifndef OGR_PMTILES_H_INCLUDED
14 : #define OGR_PMTILES_H_INCLUDED
15 :
16 : #include "gdal_priv.h"
17 : #include "ogrsf_frmts.h"
18 :
19 : #include "cpl_compressor.h"
20 : #include "cpl_vsi_virtual.h"
21 :
22 : #include "include_pmtiles.h"
23 :
24 : #include <array>
25 : #include <limits>
26 : #include <set>
27 : #include <stack>
28 :
29 : // #define DEBUG_PMTILES
30 :
31 : #define SPHERICAL_RADIUS 6378137.0
32 : #define MAX_GM (SPHERICAL_RADIUS * M_PI) // 20037508.342789244
33 :
34 : #if defined(HAVE_SQLITE) && defined(HAVE_GEOS)
35 : // Needed by mvtutils.h
36 : #define HAVE_MVT_WRITE_SUPPORT
37 : #endif
38 :
39 : constexpr int PMTILES_HEADER_LENGTH = 127;
40 :
41 : /************************************************************************/
42 : /* OGRPMTilesDataset */
43 : /************************************************************************/
44 :
45 : class OGRPMTilesDataset final : public GDALDataset
46 : {
47 : public:
48 207 : OGRPMTilesDataset() = default;
49 :
50 : ~OGRPMTilesDataset() override;
51 :
52 : bool Open(GDALOpenInfo *poOpenInfo);
53 :
54 327 : int GetLayerCount() const override
55 : {
56 327 : return static_cast<int>(m_apoLayers.size());
57 : }
58 :
59 : const OGRLayer *GetLayer(int) const override;
60 :
61 2 : const OGRSpatialReference *GetSpatialRef() const override
62 : {
63 2 : return !m_oSRS.IsEmpty() ? &m_oSRS : nullptr;
64 : }
65 :
66 3 : CPLErr GetGeoTransform(GDALGeoTransform >) const override
67 : {
68 3 : gt = m_gt;
69 3 : return !m_oSRS.IsEmpty() ? CE_None : CE_Failure;
70 : }
71 :
72 3 : inline int GetMinZoomLevel() const
73 : {
74 3 : return m_nMinZoomLevel;
75 : }
76 :
77 8 : inline int GetMaxZoomLevel() const
78 : {
79 8 : return m_nMaxZoomLevel;
80 : }
81 :
82 1540 : inline const pmtiles::headerv3 &GetHeader() const
83 : {
84 1540 : return m_sHeader;
85 : }
86 :
87 : static const char *GetCompression(uint8_t nVal);
88 :
89 : static const char *GetTileType(const pmtiles::headerv3 &sHeader);
90 :
91 7 : inline const std::string &GetMetadataContent() const
92 : {
93 7 : return m_osMetadata;
94 : }
95 :
96 687 : inline const std::string &GetMetadataFilename() const
97 : {
98 687 : return m_osMetadataFilename;
99 : }
100 :
101 681 : inline const std::string &GetClipOpenOption() const
102 : {
103 681 : return m_osClipOpenOption;
104 : }
105 :
106 : /** Return a short-lived decompressed buffer for metadata or directory
107 : * entries or nullptr in case of error.
108 : */
109 : const std::string *ReadInternal(uint64_t nOffset, uint64_t nSize,
110 : const char *pszDataType);
111 :
112 : /** Return a short-lived decompressed buffer for tile data.
113 : * or nullptr in case of error.
114 : */
115 : const std::string *ReadTileData(uint64_t nOffset, uint64_t nSize);
116 :
117 : protected:
118 : friend class GDALPMTilesRasterBand;
119 :
120 : CPLErr IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize,
121 : int nYSize, void *pData, int nBufXSize, int nBufYSize,
122 : GDALDataType eBufType, int nBandCount,
123 : BANDMAP_TYPE panBandMap, GSpacing nPixelSpace,
124 : GSpacing nLineSpace, GSpacing nBandSpace,
125 : GDALRasterIOExtraArg *psExtraArg) override;
126 :
127 : private:
128 : VSIVirtualHandleUniquePtr m_poFileUniquePtr{};
129 :
130 : VSIVirtualHandle *m_poFile = nullptr;
131 :
132 : //! PMTiles header
133 : pmtiles::headerv3 m_sHeader{};
134 :
135 : //! JSON serialized metadata
136 : std::string m_osMetadata{};
137 :
138 : //! /vsimem/ filename with the m_osMetadata content
139 : std::string m_osMetadataFilename{};
140 :
141 : //! Value of the CLIP open option
142 : std::string m_osClipOpenOption{};
143 :
144 : //! Decompressor for metadata and directories
145 : const CPLCompressor *m_psInternalDecompressor = nullptr;
146 :
147 : //! Decompressor for tile
148 : const CPLCompressor *m_psTileDataDecompressor = nullptr;
149 :
150 : //! Last raw data read by Read()
151 : std::string m_osBuffer{};
152 :
153 : //! Last uncompressed data read by Read(). Only used if compression
154 : std::string m_osDecompressedBuffer{};
155 :
156 : std::vector<std::unique_ptr<OGRLayer>> m_apoLayers{};
157 :
158 : //! Minimum zoom level got from header
159 : int m_nMinZoomLevel = 0;
160 :
161 : //! Maximum zoom level got from header
162 : int m_nMaxZoomLevel = 0;
163 :
164 : //! Zoom level got (for raster)
165 : int m_nZoomLevel = 0;
166 :
167 : //! CRS (for raster)
168 : OGRSpatialReference m_oSRS{};
169 :
170 : //! Geotransform
171 : GDALGeoTransform m_gt{};
172 :
173 : //! GeoPackage driver (for raster)
174 : GDALDriver *m_poGPKGDriver = nullptr;
175 :
176 : //! Temporary GeoPackage filename (for raster)
177 : std::string m_osTmpGPKGFilename{};
178 :
179 : //! Overview datasets
180 : std::vector<std::unique_ptr<OGRPMTilesDataset>> m_apoOverviews{};
181 :
182 : bool OpenVector(const GDALOpenInfo *poOpenInfo,
183 : const CPLJSONObject &oJsonRoot, int nZoomLevel);
184 :
185 : bool OpenRaster(int nZoomLevel);
186 :
187 : /** Return a short-lived decompressed buffer, or nullptr in case of error
188 : */
189 : const std::string *Read(const CPLCompressor *psDecompressor,
190 : uint64_t nOffset, uint64_t nSize,
191 : const char *pszDataType);
192 :
193 : CPL_DISALLOW_COPY_ASSIGN(OGRPMTilesDataset)
194 : };
195 :
196 : /************************************************************************/
197 : /* GDALPMTilesRasterBand */
198 : /************************************************************************/
199 :
200 : class GDALPMTilesRasterBand final : public GDALRasterBand
201 : {
202 : public:
203 : GDALPMTilesRasterBand(OGRPMTilesDataset *poDSIn, int nBandIn,
204 : int nBlockSize);
205 :
206 : GDALColorInterp GetColorInterpretation() override;
207 :
208 : int GetOverviewCount() override;
209 :
210 : GDALRasterBand *GetOverview(int) override;
211 :
212 : protected:
213 : CPLErr IReadBlock(int nBlockXOff, int nBlockYOff, void *pData) override;
214 :
215 : CPLErr IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize,
216 : int nYSize, void *pData, int nBufXSize, int nBufYSize,
217 : GDALDataType eBufType, GSpacing nPixelSpace,
218 : GSpacing nLineSpace,
219 : GDALRasterIOExtraArg *psExtraArg) override;
220 : };
221 :
222 : /************************************************************************/
223 : /* OGRPMTilesTileIterator */
224 : /************************************************************************/
225 :
226 : //! Iterator to browse through tiles
227 : class OGRPMTilesTileIterator
228 : {
229 : public:
230 : //! Constructor to iterate over all tiles (possibly limited to a zoom level
231 53 : explicit OGRPMTilesTileIterator(OGRPMTilesDataset *poDS,
232 : int nZoomLevel = -1)
233 53 : : m_poDS(poDS), m_nZoomLevel(nZoomLevel)
234 : {
235 53 : }
236 :
237 : //! Constructor with a window of interest in tile coordinates
238 326 : OGRPMTilesTileIterator(OGRPMTilesDataset *poDS, int nZoomLevel, int nMinX,
239 : int nMinY, int nMaxX, int nMaxY)
240 326 : : m_poDS(poDS), m_nZoomLevel(nZoomLevel), m_nMinX(nMinX),
241 326 : m_nMinY(nMinY), m_nMaxX(nMaxX), m_nMaxY(nMaxY)
242 : {
243 326 : }
244 :
245 : /** Return the (z, x, y, offset, length) of the next tile.
246 : *
247 : * If entry_zxy.offset == 0, the iteration has stopped.
248 : */
249 : pmtiles::entry_zxy GetNextTile(uint32_t *pnRunLength = nullptr);
250 :
251 : void SkipRunLength();
252 :
253 : #ifdef DEBUG_PMTILES
254 : void DumpTiles();
255 : #endif
256 :
257 : private:
258 : // Input parameters
259 : OGRPMTilesDataset *m_poDS = nullptr;
260 : int m_nZoomLevel = -1;
261 : int m_nMinX = -1;
262 : int m_nMinY = -1;
263 : int m_nMaxX = -1;
264 : int m_nMaxY = -1;
265 :
266 : // Used when iterating over tile id is inefficient
267 : int m_nCurX = -1;
268 : int m_nCurY = -1;
269 :
270 : // for sanity checks. Must be increasing when walking through entries
271 : static constexpr uint64_t INVALID_LAST_TILE_ID =
272 : std::numeric_limits<uint64_t>::max();
273 : uint64_t m_nLastTileId = INVALID_LAST_TILE_ID;
274 :
275 : // Computed values from zoom leven and min/max x/y
276 : uint64_t m_nMinTileId = std::numeric_limits<uint64_t>::max();
277 : uint64_t m_nMaxTileId = 0;
278 :
279 : bool m_bEOF = false;
280 :
281 : // State of exploration of a directory
282 : struct DirectoryContext
283 : {
284 : // Entries, either tiles (sEntry.run_length > 0) or subdiretories
285 : // (sEntry.run_length == 0)
286 : std::vector<pmtiles::entryv3> sEntries{};
287 :
288 : // Next index of sEntries[] to explore
289 : uint32_t nIdxInEntries = 0;
290 :
291 : // For tiles, value between 0 and
292 : // sEntries[nIdxInEntries].run_length - 1
293 : uint32_t nIdxInRunLength = 0;
294 : };
295 :
296 : // Stack of directories: bottom is root directory, and then we
297 : // push subdiretories we browse throw
298 : std::stack<DirectoryContext> m_aoStack{};
299 :
300 : bool LoadRootDirectory();
301 :
302 : CPL_DISALLOW_COPY_ASSIGN(OGRPMTilesTileIterator)
303 : };
304 :
305 : /************************************************************************/
306 : /* OGRPMTilesVectorLayer */
307 : /************************************************************************/
308 :
309 : class OGRPMTilesVectorLayer final
310 : : public OGRLayer,
311 : public OGRGetNextFeatureThroughRaw<OGRPMTilesVectorLayer>
312 : {
313 : public:
314 : OGRPMTilesVectorLayer(OGRPMTilesDataset *poDS, const char *pszLayerName,
315 : const CPLJSONObject &oFields,
316 : const CPLJSONArray &oAttributesFromTileStats,
317 : bool bJsonField, double dfMinX, double dfMinY,
318 : double dfMaxX, double dfMaxY,
319 : OGRwkbGeometryType eGeomType, int nZoomLevel,
320 : bool bZoomLevelFromSpatialFilter);
321 : ~OGRPMTilesVectorLayer() override;
322 :
323 : void ResetReading() override;
324 :
325 : OGRFeature *GetNextRawFeature();
326 1400 : DEFINE_GET_NEXT_FEATURE_THROUGH_RAW(OGRPMTilesVectorLayer)
327 :
328 4903 : const OGRFeatureDefn *GetLayerDefn() const override
329 : {
330 4903 : return m_poFeatureDefn;
331 : }
332 :
333 : int TestCapability(const char *) const override;
334 :
335 : OGRErr IGetExtent(int iGeomField, OGREnvelope *psExtent,
336 : bool bForce) override;
337 :
338 : OGRErr ISetSpatialFilter(int iGeomField,
339 : const OGRGeometry *poGeom) override;
340 :
341 : GIntBig GetFeatureCount(int bForce) override;
342 :
343 : OGRFeature *GetFeature(GIntBig nFID) override;
344 :
345 : static OGRwkbGeometryType GuessGeometryType(OGRPMTilesDataset *poDS,
346 : const char *pszLayerName,
347 : int nZoomLevel);
348 :
349 : private:
350 : OGRPMTilesDataset *m_poDS = nullptr;
351 : OGRFeatureDefn *m_poFeatureDefn = nullptr;
352 :
353 : //! Iterator over tiles
354 : std::unique_ptr<OGRPMTilesTileIterator> m_poTileIterator{};
355 :
356 : //! Total feature count (may over-estimate due to not applying clipping)
357 : GIntBig m_nFeatureCount = -1;
358 :
359 : //! X tile value of currently opened tile
360 : uint32_t m_nX = 0;
361 :
362 : //! Y tile value of currently opened tile
363 : uint32_t m_nY = 0;
364 :
365 : //! Offset of the currently opened tile
366 : uint64_t m_nLastTileOffset = 0;
367 :
368 : //! Uncompressed MVT tile
369 : std::string m_osTileData{};
370 :
371 : //! In-memory MVT dataset of the currently opened tile
372 : std::unique_ptr<GDALDataset> m_poTileDS{};
373 :
374 : //! Layer of m_poTileDS
375 : OGRLayer *m_poTileLayer = nullptr;
376 :
377 : //! Layer extent
378 : OGREnvelope m_sExtent{};
379 :
380 : //! Minimum X tile value corresponding to m_sFilterEnvelope
381 : int m_nFilterMinX = 0;
382 :
383 : //! Minimum Y tile value corresponding to m_sFilterEnvelope
384 : int m_nFilterMinY = 0;
385 :
386 : //! Maximum X tile value corresponding to m_sFilterEnvelope
387 : int m_nFilterMaxX = 0;
388 :
389 : //! Maximum Y tile value corresponding to m_sFilterEnvelope
390 : int m_nFilterMaxY = 0;
391 :
392 : //! Currently used zoom level
393 : int m_nZoomLevel = 0;
394 :
395 : //! Whether we should auto-adapt m_nZoomLevel from the spatial filter extent
396 : bool m_bZoomLevelAuto = false;
397 :
398 : //! Whether we should expose the tile fields in a "json" field
399 : bool m_bJsonField = false;
400 :
401 : std::unique_ptr<OGRFeature> GetNextSrcFeature();
402 : std::unique_ptr<OGRFeature> CreateFeatureFrom(OGRFeature *poSrcFeature);
403 : GIntBig GetTotalFeatureCount() const;
404 : void ExtentToTileExtent(const OGREnvelope &sEnvelope, int &nTileMinX,
405 : int &nTileMinY, int &nTileMaxX,
406 : int &nTileMaxY) const;
407 :
408 : CPL_DISALLOW_COPY_ASSIGN(OGRPMTilesVectorLayer)
409 : };
410 :
411 : const char *VSIPMTilesGetTileExtension(OGRPMTilesDataset *poDS);
412 :
413 : #ifdef HAVE_MVT_WRITE_SUPPORT
414 :
415 : /************************************************************************/
416 : /* OGRPMTilesWriterDataset */
417 : /************************************************************************/
418 :
419 : class OGRPMTilesWriterDataset final : public GDALDataset
420 : {
421 : std::unique_ptr<GDALDataset> m_poMBTilesWriterDataset{};
422 :
423 : public:
424 34 : OGRPMTilesWriterDataset() = default;
425 :
426 : ~OGRPMTilesWriterDataset() override;
427 :
428 : bool Create(const char *pszFilename, CSLConstList papszOptions);
429 :
430 : CPLErr Close(GDALProgressFunc = nullptr, void * = nullptr) override;
431 :
432 : OGRLayer *ICreateLayer(const char *pszName,
433 : const OGRGeomFieldDefn *poGeomFieldDefn,
434 : CSLConstList papszOptions) override;
435 :
436 : int TestCapability(const char *) const override;
437 : };
438 :
439 : #endif // HAVE_MVT_WRITE_SUPPORT
440 :
441 : /************************************************************************/
442 : /* HashArray() */
443 : /************************************************************************/
444 :
445 : // From https://codereview.stackexchange.com/questions/171999/specializing-stdhash-for-stdarray
446 : // We do not use std::hash<std::array<T, N>> as the name of the struct
447 : // because with gcc 5.4 we get the following error:
448 : // https://stackoverflow.com/questions/25594644/warning-specialization-of-template-in-different-namespace
449 : template <class T, size_t N> struct HashArray
450 : {
451 : CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW
452 1024 : size_t operator()(const std::array<T, N> &key) const
453 : {
454 : std::hash<T> hasher;
455 1024 : size_t result = 0;
456 17408 : for (size_t i = 0; i < N; ++i)
457 : {
458 16384 : result = result * 31 + hasher(key[i]);
459 : }
460 1024 : return result;
461 : }
462 : };
463 :
464 : #endif // OGR_PMTILES_H_INCLUDED
|
