Line data    Source code

       1             : /******************************************************************************
       2             :  *
       3             :  * Project:  OpenGIS Simple Features Reference Implementation
       4             :  * Purpose:  Implementation of PMTiles
       5             :  * Author:   Even Rouault <even.rouault at spatialys.com>
       6             :  *
       7             :  ******************************************************************************
       8             :  * Copyright (c) 2023, Planet Labs
       9             :  *
      10             :  * Permission is hereby granted, free of charge, to any person obtaining a
      11             :  * copy of this software and associated documentation files (the "Software"),
      12             :  * to deal in the Software without restriction, including without limitation
      13             :  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
      14             :  * and/or sell copies of the Software, and to permit persons to whom the
      15             :  * Software is furnished to do so, subject to the following conditions:
      16             :  *
      17             :  * The above copyright notice and this permission notice shall be included
      18             :  * in all copies or substantial portions of the Software.
      19             :  *
      20             :  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
      21             :  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
      22             :  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
      23             :  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
      24             :  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
      25             :  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
      26             :  * DEALINGS IN THE SOFTWARE.
      27             :  ****************************************************************************/
      28             : 
      29             : #include "ogr_pmtiles.h"
      30             : 
      31             : #include "cpl_json.h"
      32             : 
      33             : #include "mvtutils.h"
      34             : 
      35             : #include <math.h>
      36             : 
      37             : /************************************************************************/
      38             : /*                       ~OGRPMTilesDataset()                           */
      39             : /************************************************************************/
      40             : 
      41         192 : OGRPMTilesDataset::~OGRPMTilesDataset()
      42             : {
      43          96 :     if (!m_osMetadataFilename.empty())
      44          92 :         VSIUnlink(m_osMetadataFilename.c_str());
      45         192 : }
      46             : 
      47             : /************************************************************************/
      48             : /*                              GetLayer()                              */
      49             : /************************************************************************/
      50             : 
      51         113 : OGRLayer *OGRPMTilesDataset::GetLayer(int iLayer)
      52             : 
      53             : {
      54         113 :     if (iLayer < 0 || iLayer >= GetLayerCount())
      55          10 :         return nullptr;
      56         103 :     return m_apoLayers[iLayer].get();
      57             : }
      58             : 
      59             : /************************************************************************/
      60             : /*                     LongLatToSphericalMercator()                     */
      61             : /************************************************************************/
      62             : 
      63         184 : static void LongLatToSphericalMercator(double *x, double *y)
      64             : {
      65         184 :     double X = SPHERICAL_RADIUS * (*x) / 180 * M_PI;
      66         184 :     double Y = SPHERICAL_RADIUS * log(tan(M_PI / 4 + 0.5 * (*y) / 180 * M_PI));
      67         184 :     *x = X;
      68         184 :     *y = Y;
      69         184 : }
      70             : 
      71             : /************************************************************************/
      72             : /*                            GetCompression()                          */
      73             : /************************************************************************/
      74             : 
      75         143 : /*static*/ const char *OGRPMTilesDataset::GetCompression(uint8_t nVal)
      76             : {
      77         143 :     switch (nVal)
      78             :     {
      79           6 :         case pmtiles::COMPRESSION_UNKNOWN:
      80           6 :             return "unknown";
      81           0 :         case pmtiles::COMPRESSION_NONE:
      82           0 :             return "none";
      83         137 :         case pmtiles::COMPRESSION_GZIP:
      84         137 :             return "gzip";
      85           0 :         case pmtiles::COMPRESSION_BROTLI:
      86           0 :             return "brotli";
      87           0 :         case pmtiles::COMPRESSION_ZSTD:
      88           0 :             return "zstd";
      89           0 :         default:
      90           0 :             break;
      91             :     }
      92           0 :     return CPLSPrintf("invalid (%d)", nVal);
      93             : }
      94             : 
      95             : /************************************************************************/
      96             : /*                           GetTileType()                              */
      97             : /************************************************************************/
      98             : 
      99             : /* static */
     100           4 : const char *OGRPMTilesDataset::GetTileType(const pmtiles::headerv3 &sHeader)
     101             : {
     102           4 :     switch (sHeader.tile_type)
     103             :     {
     104           0 :         case pmtiles::TILETYPE_UNKNOWN:
     105           0 :             return "unknown";
     106           0 :         case pmtiles::TILETYPE_PNG:
     107           0 :             return "PNG";
     108           0 :         case pmtiles::TILETYPE_JPEG:
     109           0 :             return "JPEG";
     110           0 :         case pmtiles::TILETYPE_WEBP:
     111           0 :             return "WEBP";
     112           4 :         case pmtiles::TILETYPE_MVT:
     113           4 :             return "MVT";
     114           0 :         default:
     115           0 :             break;
     116             :     }
     117           0 :     return CPLSPrintf("invalid (%d)", sHeader.tile_type);
     118             : }
     119             : 
     120             : /************************************************************************/
     121             : /*                                Open()                                */
     122             : /************************************************************************/
     123             : 
     124          96 : bool OGRPMTilesDataset::Open(GDALOpenInfo *poOpenInfo)
     125             : {
     126          96 :     if (!poOpenInfo->fpL || poOpenInfo->nHeaderBytes < 127)
     127           2 :         return false;
     128             : 
     129          94 :     SetDescription(poOpenInfo->pszFilename);
     130             : 
     131             :     // Borrow file handle
     132          94 :     m_poFile.reset(poOpenInfo->fpL);
     133          94 :     poOpenInfo->fpL = nullptr;
     134             : 
     135             :     // Deserizalize header
     136         188 :     std::string osHeader;
     137          94 :     osHeader.assign(reinterpret_cast<const char *>(poOpenInfo->pabyHeader),
     138          94 :                     127);
     139             :     try
     140             :     {
     141          94 :         m_sHeader = pmtiles::deserialize_header(osHeader);
     142             :     }
     143           1 :     catch (const std::exception &)
     144             :     {
     145           1 :         return false;
     146             :     }
     147             : 
     148             :     // Check tile type
     149          93 :     const bool bAcceptAnyTileType = CPLTestBool(CSLFetchNameValueDef(
     150          93 :         poOpenInfo->papszOpenOptions, "ACCEPT_ANY_TILE_TYPE", "NO"));
     151          93 :     if (bAcceptAnyTileType)
     152             :     {
     153             :         // do nothing. Internal use only by /vsipmtiles/
     154             :     }
     155          42 :     else if (m_sHeader.tile_type != pmtiles::TILETYPE_MVT)
     156             :     {
     157           0 :         CPLError(CE_Failure, CPLE_AppDefined,
     158             :                  "Tile type %s not handled by the driver",
     159           0 :                  GetTileType(m_sHeader));
     160           0 :         return false;
     161             :     }
     162             : 
     163             :     // Check compression method for metadata and directories
     164          93 :     CPLDebugOnly("PMTiles", "internal_compression = %s",
     165             :                  GetCompression(m_sHeader.internal_compression));
     166             : 
     167          93 :     if (m_sHeader.internal_compression == pmtiles::COMPRESSION_GZIP)
     168             :     {
     169          93 :         m_psInternalDecompressor = CPLGetDecompressor("gzip");
     170             :     }
     171           0 :     else if (m_sHeader.internal_compression == pmtiles::COMPRESSION_ZSTD)
     172             :     {
     173           0 :         m_psInternalDecompressor = CPLGetDecompressor("zstd");
     174           0 :         if (m_psInternalDecompressor == nullptr)
     175             :         {
     176           0 :             CPLError(CE_Failure, CPLE_AppDefined,
     177             :                      "File %s requires ZSTD decompression, but not available "
     178             :                      "in this GDAL build",
     179             :                      poOpenInfo->pszFilename);
     180           0 :             return false;
     181             :         }
     182             :     }
     183           0 :     else if (m_sHeader.internal_compression != pmtiles::COMPRESSION_NONE)
     184             :     {
     185           0 :         CPLError(CE_Failure, CPLE_AppDefined,
     186             :                  "Unhandled internal_compression = %s",
     187           0 :                  GetCompression(m_sHeader.internal_compression));
     188           0 :         return false;
     189             :     }
     190             : 
     191             :     // Check compression for tile data
     192          93 :     if (!CPLTestBool(CSLFetchNameValueDef(poOpenInfo->papszOpenOptions,
     193             :                                           "DECOMPRESS_TILES", "YES")))
     194             :     {
     195             :         // do nothing. Internal use only by /vsipmtiles/
     196             :     }
     197             :     else
     198             :     {
     199          42 :         CPLDebugOnly("PMTiles", "tile_compression = %s",
     200             :                      GetCompression(m_sHeader.tile_compression));
     201             : 
     202          42 :         if (m_sHeader.tile_compression == pmtiles::COMPRESSION_UNKNOWN)
     203             :         {
     204             :             // Python pmtiles-convert generates this. The MVT driver can autodetect
     205             :             // uncompressed and GZip-compressed tiles automatically.
     206             :         }
     207          38 :         else if (m_sHeader.tile_compression == pmtiles::COMPRESSION_GZIP)
     208             :         {
     209          38 :             m_psTileDataDecompressor = CPLGetDecompressor("gzip");
     210             :         }
     211           0 :         else if (m_sHeader.tile_compression == pmtiles::COMPRESSION_ZSTD)
     212             :         {
     213           0 :             m_psTileDataDecompressor = CPLGetDecompressor("zstd");
     214           0 :             if (m_psTileDataDecompressor == nullptr)
     215             :             {
     216           0 :                 CPLError(
     217             :                     CE_Failure, CPLE_AppDefined,
     218             :                     "File %s requires ZSTD decompression, but not available "
     219             :                     "in this GDAL build",
     220             :                     poOpenInfo->pszFilename);
     221           0 :                 return false;
     222             :             }
     223             :         }
     224           0 :         else if (m_sHeader.tile_compression != pmtiles::COMPRESSION_NONE)
     225             :         {
     226           0 :             CPLError(CE_Failure, CPLE_AppDefined,
     227             :                      "Unhandled tile_compression = %s",
     228           0 :                      GetCompression(m_sHeader.tile_compression));
     229           0 :             return false;
     230             :         }
     231             :     }
     232             : 
     233             :     // Read metadata
     234             :     const auto *posMetadata =
     235          93 :         ReadInternal(m_sHeader.json_metadata_offset,
     236             :                      m_sHeader.json_metadata_bytes, "metadata");
     237          93 :     if (!posMetadata)
     238           1 :         return false;
     239          92 :     CPLDebugOnly("PMTiles", "Metadata = %s", posMetadata->c_str());
     240          92 :     m_osMetadata = *posMetadata;
     241             : 
     242          92 :     m_osMetadataFilename = CPLSPrintf("/vsimem/pmtiles/metadata_%p.json", this);
     243          92 :     VSIFCloseL(VSIFileFromMemBuffer(m_osMetadataFilename.c_str(),
     244          92 :                                     reinterpret_cast<GByte *>(&m_osMetadata[0]),
     245          92 :                                     m_osMetadata.size(), false));
     246             : 
     247         184 :     CPLJSONDocument oJsonDoc;
     248          92 :     if (!oJsonDoc.LoadMemory(m_osMetadata))
     249             :     {
     250           0 :         CPLError(CE_Failure, CPLE_AppDefined, "Cannot parse metadata");
     251           0 :         return false;
     252             :     }
     253             : 
     254         184 :     auto oJsonRoot = oJsonDoc.GetRoot();
     255        1117 :     for (const auto &oChild : oJsonRoot.GetChildren())
     256             :     {
     257        1025 :         if (oChild.GetType() == CPLJSONObject::Type::String)
     258             :         {
     259         905 :             if (oChild.GetName() == "json")
     260             :             {
     261             :                 // Tippecanoe metadata includes a "json" item, which is a
     262             :                 // serialized JSON object with vector_layers[] and layers[]
     263             :                 // arrays we are interested in later.
     264             :                 // so use "json" content as the new root
     265          31 :                 if (!oJsonDoc.LoadMemory(oChild.ToString()))
     266             :                 {
     267           0 :                     CPLError(CE_Failure, CPLE_AppDefined,
     268             :                              "Cannot parse 'json' metadata item");
     269           0 :                     return false;
     270             :                 }
     271          31 :                 oJsonRoot = oJsonDoc.GetRoot();
     272             :             }
     273             :             // Tippecanoe generates a "strategies" member with serialized JSON
     274         874 :             else if (oChild.GetName() != "strategies")
     275             :             {
     276         874 :                 SetMetadataItem(oChild.GetName().c_str(),
     277        1748 :                                 oChild.ToString().c_str());
     278             :             }
     279             :         }
     280             :     }
     281             : 
     282          92 :     double dfMinX = m_sHeader.min_lon_e7 / 10e6;
     283          92 :     double dfMinY = m_sHeader.min_lat_e7 / 10e6;
     284          92 :     double dfMaxX = m_sHeader.max_lon_e7 / 10e6;
     285          92 :     double dfMaxY = m_sHeader.max_lat_e7 / 10e6;
     286          92 :     LongLatToSphericalMercator(&dfMinX, &dfMinY);
     287          92 :     LongLatToSphericalMercator(&dfMaxX, &dfMaxY);
     288             : 
     289          92 :     m_nMinZoomLevel = m_sHeader.min_zoom;
     290          92 :     m_nMaxZoomLevel = m_sHeader.max_zoom;
     291          92 :     if (m_nMinZoomLevel > m_nMaxZoomLevel)
     292             :     {
     293           1 :         CPLError(CE_Failure, CPLE_AppDefined, "min_zoom(=%d) > max_zoom(=%d)",
     294             :                  m_nMinZoomLevel, m_nMaxZoomLevel);
     295           1 :         return false;
     296             :     }
     297          91 :     if (m_nMinZoomLevel > 30)
     298             :     {
     299           1 :         CPLError(CE_Warning, CPLE_AppDefined, "Clamping min_zoom from %d to %d",
     300             :                  m_nMinZoomLevel, 30);
     301           1 :         m_nMinZoomLevel = 30;
     302             :     }
     303          91 :     if (m_nMaxZoomLevel > 30)
     304             :     {
     305           1 :         CPLError(CE_Warning, CPLE_AppDefined, "Clamping max_zoom from %d to %d",
     306             :                  m_nMaxZoomLevel, 30);
     307           1 :         m_nMaxZoomLevel = 30;
     308             :     }
     309             : 
     310          91 :     if (bAcceptAnyTileType)
     311          50 :         return true;
     312             : 
     313             :     // If using the pmtiles go utility, vector_layers and tilestats are
     314             :     // moved from Tippecanoe's json metadata item to the root element.
     315         123 :     CPLJSONArray oVectorLayers = oJsonRoot.GetArray("vector_layers");
     316          41 :     if (oVectorLayers.Size() == 0)
     317             :     {
     318           5 :         CPLError(CE_Failure, CPLE_AppDefined,
     319             :                  "Missing vector_layers[] metadata");
     320           5 :         return false;
     321             :     }
     322             : 
     323         108 :     CPLJSONArray oTileStatLayers = oJsonRoot.GetArray("tilestats/layers");
     324             : 
     325             :     const int nZoomLevel =
     326          36 :         atoi(CSLFetchNameValueDef(poOpenInfo->papszOpenOptions, "ZOOM_LEVEL",
     327          36 :                                   CPLSPrintf("%d", m_nMaxZoomLevel)));
     328          36 :     if (nZoomLevel < m_nMinZoomLevel || nZoomLevel > m_nMaxZoomLevel)
     329             :     {
     330           2 :         CPLError(CE_Failure, CPLE_AppDefined,
     331             :                  "Invalid zoom level. Should be in [%d,%d] range",
     332             :                  m_nMinZoomLevel, m_nMaxZoomLevel);
     333           2 :         return false;
     334             :     }
     335          34 :     SetMetadataItem("ZOOM_LEVEL", CPLSPrintf("%d", nZoomLevel));
     336             : 
     337             :     m_osClipOpenOption =
     338          34 :         CSLFetchNameValueDef(poOpenInfo->papszOpenOptions, "CLIP", "");
     339             : 
     340          68 :     const bool bZoomLevelFromSpatialFilter = CPLFetchBool(
     341          34 :         poOpenInfo->papszOpenOptions, "ZOOM_LEVEL_AUTO",
     342          34 :         CPLTestBool(CPLGetConfigOption("MVT_ZOOM_LEVEL_AUTO", "NO")));
     343             :     const bool bJsonField =
     344          34 :         CPLFetchBool(poOpenInfo->papszOpenOptions, "JSON_FIELD", false);
     345             : 
     346          75 :     for (int i = 0; i < oVectorLayers.Size(); i++)
     347             :     {
     348         123 :         CPLJSONObject oId = oVectorLayers[i].GetObj("id");
     349          41 :         if (oId.IsValid() && oId.GetType() == CPLJSONObject::Type::String)
     350             :         {
     351          41 :             OGRwkbGeometryType eGeomType = wkbUnknown;
     352          41 :             if (oTileStatLayers.IsValid())
     353             :             {
     354          39 :                 eGeomType = OGRMVTFindGeomTypeFromTileStat(
     355          78 :                     oTileStatLayers, oId.ToString().c_str());
     356             :             }
     357          41 :             if (eGeomType == wkbUnknown)
     358             :             {
     359           2 :                 eGeomType = OGRPMTilesVectorLayer::GuessGeometryType(
     360           4 :                     this, oId.ToString().c_str(), nZoomLevel);
     361             :             }
     362             : 
     363         123 :             CPLJSONObject oFields = oVectorLayers[i].GetObj("fields");
     364             :             CPLJSONArray oAttributesFromTileStats =
     365             :                 OGRMVTFindAttributesFromTileStat(oTileStatLayers,
     366          82 :                                                  oId.ToString().c_str());
     367             : 
     368          41 :             m_apoLayers.push_back(std::make_unique<OGRPMTilesVectorLayer>(
     369          82 :                 this, oId.ToString().c_str(), oFields, oAttributesFromTileStats,
     370             :                 bJsonField, dfMinX, dfMinY, dfMaxX, dfMaxY, eGeomType,
     371             :                 nZoomLevel, bZoomLevelFromSpatialFilter));
     372             :         }
     373             :     }
     374             : 
     375          34 :     return true;
     376             : }
     377             : 
     378             : /************************************************************************/
     379             : /*                              Read()                                  */
     380             : /************************************************************************/
     381             : 
     382         798 : const std::string *OGRPMTilesDataset::Read(const CPLCompressor *psDecompressor,
     383             :                                            uint64_t nOffset, uint64_t nSize,
     384             :                                            const char *pszDataType)
     385             : {
     386         798 :     if (nSize > 10 * 1024 * 1024)
     387             :     {
     388           0 :         CPLError(CE_Failure, CPLE_AppDefined,
     389             :                  "Too large amount of %s to read: " CPL_FRMT_GUIB
     390             :                  " bytes at offset " CPL_FRMT_GUIB,
     391             :                  pszDataType, static_cast<GUIntBig>(nSize),
     392             :                  static_cast<GUIntBig>(nOffset));
     393           0 :         return nullptr;
     394             :     }
     395         798 :     m_osBuffer.resize(static_cast<size_t>(nSize));
     396         798 :     m_poFile->Seek(nOffset, SEEK_SET);
     397         798 :     if (m_poFile->Read(&m_osBuffer[0], m_osBuffer.size(), 1) != 1)
     398             :     {
     399           1 :         CPLError(CE_Failure, CPLE_AppDefined,
     400             :                  "Cannot read %s of length %u at offset " CPL_FRMT_GUIB,
     401             :                  pszDataType, unsigned(nSize), static_cast<GUIntBig>(nOffset));
     402           1 :         return nullptr;
     403             :     }
     404             : 
     405         797 :     if (psDecompressor)
     406             :     {
     407         684 :         m_osDecompressedBuffer.resize(32 + 16 * m_osBuffer.size());
     408         684 :         for (int iTry = 0; iTry < 2; ++iTry)
     409             :         {
     410         684 :             void *pOutputData = &m_osDecompressedBuffer[0];
     411         684 :             size_t nOutputSize = m_osDecompressedBuffer.size();
     412         684 :             if (!psDecompressor->pfnFunc(m_osBuffer.data(), m_osBuffer.size(),
     413             :                                          &pOutputData, &nOutputSize, nullptr,
     414         684 :                                          psDecompressor->user_data))
     415             :             {
     416           0 :                 if (iTry == 0)
     417             :                 {
     418           0 :                     pOutputData = nullptr;
     419           0 :                     nOutputSize = 0;
     420           0 :                     if (psDecompressor->pfnFunc(
     421           0 :                             m_osBuffer.data(), m_osBuffer.size(), &pOutputData,
     422           0 :                             &nOutputSize, nullptr, psDecompressor->user_data))
     423             :                     {
     424           0 :                         CPLDebug("PMTiles",
     425             :                                  "Buffer of size %u uncompresses to %u bytes",
     426             :                                  unsigned(nSize), unsigned(nOutputSize));
     427           0 :                         m_osDecompressedBuffer.resize(nOutputSize);
     428           0 :                         continue;
     429             :                     }
     430             :                 }
     431             : 
     432           0 :                 CPLError(CE_Failure, CPLE_AppDefined,
     433             :                          "Cannot decompress %s of length %u at "
     434             :                          "offset " CPL_FRMT_GUIB,
     435             :                          pszDataType, unsigned(nSize),
     436             :                          static_cast<GUIntBig>(nOffset));
     437           0 :                 return nullptr;
     438             :             }
     439         684 :             m_osDecompressedBuffer.resize(nOutputSize);
     440         684 :             break;
     441             :         }
     442         684 :         return &m_osDecompressedBuffer;
     443             :     }
     444             :     else
     445             :     {
     446         113 :         return &m_osBuffer;
     447             :     }
     448             : }
     449             : 
     450             : /************************************************************************/
     451             : /*                              ReadInternal()                          */
     452             : /************************************************************************/
     453             : 
     454         549 : const std::string *OGRPMTilesDataset::ReadInternal(uint64_t nOffset,
     455             :                                                    uint64_t nSize,
     456             :                                                    const char *pszDataType)
     457             : {
     458         549 :     return Read(m_psInternalDecompressor, nOffset, nSize, pszDataType);
     459             : }
     460             : 
     461             : /************************************************************************/
     462             : /*                              ReadTileData()                          */
     463             : /************************************************************************/
     464             : 
     465         249 : const std::string *OGRPMTilesDataset::ReadTileData(uint64_t nOffset,
     466             :                                                    uint64_t nSize)
     467             : {
     468         249 :     return Read(m_psTileDataDecompressor, nOffset, nSize, "tile data");
     469             : }