Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: OpenGIS Simple Features Reference Implementation
4 : * Purpose: Implementation of PMTiles
5 : * Author: Even Rouault <even.rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2023, Planet Labs
9 : *
10 : * Permission is hereby granted, free of charge, to any person obtaining a
11 : * copy of this software and associated documentation files (the "Software"),
12 : * to deal in the Software without restriction, including without limitation
13 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 : * and/or sell copies of the Software, and to permit persons to whom the
15 : * Software is furnished to do so, subject to the following conditions:
16 : *
17 : * The above copyright notice and this permission notice shall be included
18 : * in all copies or substantial portions of the Software.
19 : *
20 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 : * DEALINGS IN THE SOFTWARE.
27 : ****************************************************************************/
28 :
29 : #ifndef OGR_PMTILES_H_INCLUDED
30 : #define OGR_PMTILES_H_INCLUDED
31 :
32 : #include "gdal_priv.h"
33 : #include "ogrsf_frmts.h"
34 :
35 : #include "cpl_compressor.h"
36 : #include "cpl_vsi_virtual.h"
37 :
38 : #include "include_pmtiles.h"
39 :
40 : #include <limits>
41 : #include <set>
42 : #include <stack>
43 :
44 : // #define DEBUG_PMTILES
45 :
46 : #define SPHERICAL_RADIUS 6378137.0
47 : #define MAX_GM (SPHERICAL_RADIUS * M_PI) // 20037508.342789244
48 :
49 : #if defined(HAVE_SQLITE) && defined(HAVE_GEOS)
50 : // Needed by mvtutils.h
51 : #define HAVE_MVT_WRITE_SUPPORT
52 : #endif
53 :
54 : /************************************************************************/
55 : /* OGRPMTilesDataset */
56 : /************************************************************************/
57 :
58 : class OGRPMTilesDataset final : public GDALDataset
59 : {
60 : public:
61 96 : OGRPMTilesDataset() = default;
62 :
63 : ~OGRPMTilesDataset() override;
64 :
65 : bool Open(GDALOpenInfo *poOpenInfo);
66 :
67 230 : int GetLayerCount() override
68 : {
69 230 : return static_cast<int>(m_apoLayers.size());
70 : }
71 :
72 : OGRLayer *GetLayer(int) override;
73 :
74 3 : inline int GetMinZoomLevel() const
75 : {
76 3 : return m_nMinZoomLevel;
77 : }
78 :
79 8 : inline int GetMaxZoomLevel() const
80 : {
81 8 : return m_nMaxZoomLevel;
82 : }
83 :
84 1141 : inline const pmtiles::headerv3 &GetHeader() const
85 : {
86 1141 : return m_sHeader;
87 : }
88 :
89 : static const char *GetCompression(uint8_t nVal);
90 :
91 : static const char *GetTileType(const pmtiles::headerv3 &sHeader);
92 :
93 6 : inline const std::string &GetMetadataContent() const
94 : {
95 6 : return m_osMetadata;
96 : }
97 :
98 673 : inline const std::string &GetMetadataFilename() const
99 : {
100 673 : return m_osMetadataFilename;
101 : }
102 :
103 667 : inline const std::string &GetClipOpenOption() const
104 : {
105 667 : return m_osClipOpenOption;
106 : }
107 :
108 : /** Return a short-lived decompressed buffer for metadata or directory
109 : * entries or nullptr in case of error.
110 : */
111 : const std::string *ReadInternal(uint64_t nOffset, uint64_t nSize,
112 : const char *pszDataType);
113 :
114 : /** Return a short-lived decompressed buffer for tile data.
115 : * or nullptr in case of error.
116 : */
117 : const std::string *ReadTileData(uint64_t nOffset, uint64_t nSize);
118 :
119 : private:
120 : VSIVirtualHandleUniquePtr m_poFile{};
121 :
122 : //! PMTiles header
123 : pmtiles::headerv3 m_sHeader{};
124 :
125 : //! JSON serialized metadata
126 : std::string m_osMetadata{};
127 :
128 : //! /vsimem/ filename with the m_osMetadata content
129 : std::string m_osMetadataFilename{};
130 :
131 : //! Value of the CLIP open option
132 : std::string m_osClipOpenOption{};
133 :
134 : //! Decompressor for metadata and directories
135 : const CPLCompressor *m_psInternalDecompressor = nullptr;
136 :
137 : //! Decompressor for tile
138 : const CPLCompressor *m_psTileDataDecompressor = nullptr;
139 :
140 : //! Last raw data read by Read()
141 : std::string m_osBuffer{};
142 :
143 : //! Last uncompressed data read by Read(). Only used if compression
144 : std::string m_osDecompressedBuffer{};
145 :
146 : std::vector<std::unique_ptr<OGRLayer>> m_apoLayers{};
147 :
148 : //! Minimum zoom level got from header
149 : int m_nMinZoomLevel = 0;
150 :
151 : //! Maximum zoom level got from header
152 : int m_nMaxZoomLevel = 0;
153 :
154 : /** Return a short-lived decompressed buffer, or nullptr in case of error
155 : */
156 : const std::string *Read(const CPLCompressor *psDecompressor,
157 : uint64_t nOffset, uint64_t nSize,
158 : const char *pszDataType);
159 :
160 : CPL_DISALLOW_COPY_ASSIGN(OGRPMTilesDataset)
161 : };
162 :
163 : /************************************************************************/
164 : /* OGRPMTilesTileIterator */
165 : /************************************************************************/
166 :
167 : //! Iterator to browse through tiles
168 : class OGRPMTilesTileIterator
169 : {
170 : public:
171 : //! Constructor to iterate over all tiles (possibly limited to a zoom level
172 15 : explicit OGRPMTilesTileIterator(OGRPMTilesDataset *poDS,
173 : int nZoomLevel = -1)
174 15 : : m_poDS(poDS), m_nZoomLevel(nZoomLevel)
175 : {
176 15 : }
177 :
178 : //! Constructor with a window of interest in tile coordinates
179 237 : OGRPMTilesTileIterator(OGRPMTilesDataset *poDS, int nZoomLevel, int nMinX,
180 : int nMinY, int nMaxX, int nMaxY)
181 237 : : m_poDS(poDS), m_nZoomLevel(nZoomLevel), m_nMinX(nMinX),
182 237 : m_nMinY(nMinY), m_nMaxX(nMaxX), m_nMaxY(nMaxY)
183 : {
184 237 : }
185 :
186 : /** Return the (z, x, y, offset, length) of the next tile.
187 : *
188 : * If entry_zxy.offset == 0, the iteration has stopped.
189 : */
190 : pmtiles::entry_zxy GetNextTile(uint32_t *pnRunLength = nullptr);
191 :
192 : void SkipRunLength();
193 :
194 : #ifdef DEBUG_PMTILES
195 : void DumpTiles();
196 : #endif
197 :
198 : private:
199 : // Input parameters
200 : OGRPMTilesDataset *m_poDS = nullptr;
201 : int m_nZoomLevel = -1;
202 : int m_nMinX = -1;
203 : int m_nMinY = -1;
204 : int m_nMaxX = -1;
205 : int m_nMaxY = -1;
206 :
207 : // Used when iterating over tile id is inefficient
208 : int m_nCurX = -1;
209 : int m_nCurY = -1;
210 :
211 : // for sanity checks. Must be increasing when walking through entries
212 : static constexpr uint64_t INVALID_LAST_TILE_ID =
213 : std::numeric_limits<uint64_t>::max();
214 : uint64_t m_nLastTileId = INVALID_LAST_TILE_ID;
215 :
216 : // Computed values from zoom leven and min/max x/y
217 : uint64_t m_nMinTileId = std::numeric_limits<uint64_t>::max();
218 : uint64_t m_nMaxTileId = 0;
219 :
220 : bool m_bEOF = false;
221 :
222 : // State of exploration of a directory
223 : struct DirectoryContext
224 : {
225 : // Entries, either tiles (sEntry.run_length > 0) or subdiretories
226 : // (sEntry.run_length == 0)
227 : std::vector<pmtiles::entryv3> sEntries{};
228 :
229 : // Next index of sEntries[] to explore
230 : uint32_t nIdxInEntries = 0;
231 :
232 : // For tiles, value between 0 and
233 : // sEntries[nIdxInEntries].run_length - 1
234 : uint32_t nIdxInRunLength = 0;
235 : };
236 :
237 : // Stack of directories: bottom is root directory, and then we
238 : // push subdiretories we browse throw
239 : std::stack<DirectoryContext> m_aoStack{};
240 :
241 : bool LoadRootDirectory();
242 :
243 : CPL_DISALLOW_COPY_ASSIGN(OGRPMTilesTileIterator)
244 : };
245 :
246 : /************************************************************************/
247 : /* OGRPMTilesVectorLayer */
248 : /************************************************************************/
249 :
250 : class OGRPMTilesVectorLayer final
251 : : public OGRLayer,
252 : public OGRGetNextFeatureThroughRaw<OGRPMTilesVectorLayer>
253 : {
254 : public:
255 : OGRPMTilesVectorLayer(OGRPMTilesDataset *poDS, const char *pszLayerName,
256 : const CPLJSONObject &oFields,
257 : const CPLJSONArray &oAttributesFromTileStats,
258 : bool bJsonField, double dfMinX, double dfMinY,
259 : double dfMaxX, double dfMaxY,
260 : OGRwkbGeometryType eGeomType, int nZoomLevel,
261 : bool bZoomLevelFromSpatialFilter);
262 : ~OGRPMTilesVectorLayer();
263 :
264 : void ResetReading() override;
265 :
266 : OGRFeature *GetNextRawFeature();
267 1330 : DEFINE_GET_NEXT_FEATURE_THROUGH_RAW(OGRPMTilesVectorLayer)
268 :
269 7134 : OGRFeatureDefn *GetLayerDefn() override
270 : {
271 7134 : return m_poFeatureDefn;
272 : }
273 :
274 : int TestCapability(const char *) override;
275 :
276 : OGRErr GetExtent(OGREnvelope *psExtent, int bForce) override;
277 :
278 23 : OGRErr GetExtent(int iGeomField, OGREnvelope *psExtent, int bForce) override
279 : {
280 23 : return OGRLayer::GetExtent(iGeomField, psExtent, bForce);
281 : }
282 :
283 : void SetSpatialFilter(OGRGeometry *) override;
284 :
285 52 : void SetSpatialFilter(int iGeomField, OGRGeometry *poGeom) override
286 : {
287 52 : OGRLayer::SetSpatialFilter(iGeomField, poGeom);
288 52 : }
289 :
290 : GIntBig GetFeatureCount(int bForce) override;
291 :
292 : OGRFeature *GetFeature(GIntBig nFID) override;
293 :
294 : static OGRwkbGeometryType GuessGeometryType(OGRPMTilesDataset *poDS,
295 : const char *pszLayerName,
296 : int nZoomLevel);
297 :
298 : private:
299 : OGRPMTilesDataset *m_poDS = nullptr;
300 : OGRFeatureDefn *m_poFeatureDefn = nullptr;
301 :
302 : //! Iterator over tiles
303 : std::unique_ptr<OGRPMTilesTileIterator> m_poTileIterator{};
304 :
305 : //! Total feature count (may over-estimate due to not applying clipping)
306 : GIntBig m_nFeatureCount = -1;
307 :
308 : //! X tile value of currently opened tile
309 : uint32_t m_nX = 0;
310 :
311 : //! Y tile value of currently opened tile
312 : uint32_t m_nY = 0;
313 :
314 : //! Offset of the currently opened tile
315 : uint64_t m_nLastTileOffset = 0;
316 :
317 : //! Uncompressed MVT tile
318 : std::string m_osTileData{};
319 :
320 : //! In-memory MVT dataset of the currently opened tile
321 : std::unique_ptr<GDALDataset> m_poTileDS{};
322 :
323 : //! Layer of m_poTileDS
324 : OGRLayer *m_poTileLayer = nullptr;
325 :
326 : //! Layer extent
327 : OGREnvelope m_sExtent{};
328 :
329 : //! Minimum X tile value corresponding to m_sFilterEnvelope
330 : int m_nFilterMinX = 0;
331 :
332 : //! Minimum Y tile value corresponding to m_sFilterEnvelope
333 : int m_nFilterMinY = 0;
334 :
335 : //! Maximum X tile value corresponding to m_sFilterEnvelope
336 : int m_nFilterMaxX = 0;
337 :
338 : //! Maximum Y tile value corresponding to m_sFilterEnvelope
339 : int m_nFilterMaxY = 0;
340 :
341 : //! Currently used zoom level
342 : int m_nZoomLevel = 0;
343 :
344 : //! Whether we should auto-adapt m_nZoomLevel from the spatial filter extent
345 : bool m_bZoomLevelAuto = false;
346 :
347 : //! Whether we should expose the tile fields in a "json" field
348 : bool m_bJsonField = false;
349 :
350 : std::unique_ptr<OGRFeature> GetNextSrcFeature();
351 : std::unique_ptr<OGRFeature> CreateFeatureFrom(OGRFeature *poSrcFeature);
352 : GIntBig GetTotalFeatureCount() const;
353 : void ExtentToTileExtent(const OGREnvelope &sEnvelope, int &nTileMinX,
354 : int &nTileMinY, int &nTileMaxX,
355 : int &nTileMaxY) const;
356 :
357 : CPL_DISALLOW_COPY_ASSIGN(OGRPMTilesVectorLayer)
358 : };
359 :
360 : #ifdef HAVE_MVT_WRITE_SUPPORT
361 :
362 : /************************************************************************/
363 : /* OGRPMTilesWriterDataset */
364 : /************************************************************************/
365 :
366 : class OGRPMTilesWriterDataset final : public GDALDataset
367 : {
368 : std::unique_ptr<GDALDataset> m_poMBTilesWriterDataset{};
369 :
370 : public:
371 34 : OGRPMTilesWriterDataset() = default;
372 :
373 : ~OGRPMTilesWriterDataset() override;
374 :
375 : bool Create(const char *pszFilename, CSLConstList papszOptions);
376 :
377 : CPLErr Close() override;
378 :
379 : OGRLayer *ICreateLayer(const char *pszName,
380 : const OGRGeomFieldDefn *poGeomFieldDefn,
381 : CSLConstList papszOptions) override;
382 :
383 : int TestCapability(const char *) override;
384 : };
385 :
386 : #endif // HAVE_MVT_WRITE_SUPPORT
387 :
388 : #endif // OGR_PMTILES_H_INCLUDED
|
