Line data    Source code

       1             : /******************************************************************************
       2             :  *
       3             :  * Project:  GDAL
       4             :  * Purpose:  Zarr driver
       5             :  * Author:   Even Rouault <even dot rouault at spatialys.com>
       6             :  *
       7             :  ******************************************************************************
       8             :  * Copyright (c) 2021, Even Rouault <even dot rouault at spatialys.com>
       9             :  *
      10             :  * SPDX-License-Identifier: MIT
      11             :  ****************************************************************************/
      12             : 
      13             : #include "zarrdrivercore.h"
      14             : #include "gdal_frmts.h"
      15             : #include "gdalplugindriverproxy.h"
      16             : 
      17             : #include "cpl_string.h"
      18             : #include "cpl_vsi_virtual.h"
      19             : #include "vsikerchunk.h"
      20             : #include "vsikerchunk_inline.hpp"
      21             : 
      22             : /************************************************************************/
      23             : /*                    CheckExistenceOfOneZarrFile()                     */
      24             : /************************************************************************/
      25             : 
      26        3397 : static bool CheckExistenceOfOneZarrFile(const char *pszFilename)
      27             : {
      28             : 
      29             :     CPLString osMDFilename =
      30        6794 :         CPLFormFilenameSafe(pszFilename, ".zarray", nullptr);
      31        3397 :     if (VSIFilesystemHandler::OpenStatic(osMDFilename, "rb"))
      32         632 :         return true;
      33             : 
      34        2765 :     osMDFilename = CPLFormFilenameSafe(pszFilename, ".zgroup", nullptr);
      35        2765 :     if (VSIFilesystemHandler::OpenStatic(osMDFilename, "rb"))
      36         438 :         return true;
      37             : 
      38             :     // Zarr V3
      39        2327 :     osMDFilename = CPLFormFilenameSafe(pszFilename, "zarr.json", nullptr);
      40        2327 :     if (VSIFilesystemHandler::OpenStatic(osMDFilename, "rb"))
      41        1950 :         return true;
      42             : 
      43         377 :     return false;
      44             : }
      45             : 
      46             : /************************************************************************/
      47             : /*                    ZARRIsLikelyKerchunkJSONRef()                     */
      48             : /************************************************************************/
      49             : 
      50       63046 : bool ZARRIsLikelyKerchunkJSONRef(const GDALOpenInfo *poOpenInfo)
      51             : {
      52       66205 :     if (poOpenInfo->nHeaderBytes > 0 && poOpenInfo->eAccess == GA_ReadOnly &&
      53        3159 :         (poOpenInfo->IsExtensionEqualToCI("json") ||
      54             :          // e.g. like in https://noaa-nodd-kerchunk-pds.s3.amazonaws.com/nos/cbofs/cbofs.fields.best.nc.zarr
      55        1490 :          poOpenInfo->IsExtensionEqualToCI("zarr")))
      56             :     {
      57         188 :         const char *pszHeader =
      58             :             reinterpret_cast<const char *>(poOpenInfo->pabyHeader);
      59         188 :         if (ZARRIsLikelyStreamableKerchunkJSONRefContent(
      60         376 :                 std::string_view(pszHeader, poOpenInfo->nHeaderBytes)))
      61             :         {
      62          57 :             return true;
      63             :         }
      64             :     }
      65       62989 :     return false;
      66             : }
      67             : 
      68             : /************************************************************************/
      69             : /*                         ZARRDriverIdentify()                         */
      70             : /************************************************************************/
      71             : 
      72       61518 : int ZARRDriverIdentify(GDALOpenInfo *poOpenInfo)
      73             : 
      74             : {
      75       61518 :     const std::string_view osvFilename(poOpenInfo->pszFilename);
      76      122876 :     if (cpl::starts_with(osvFilename, "ZARR:") ||
      77      122876 :         cpl::starts_with(osvFilename, "ZARR_DUMMY:"))
      78             :     {
      79         162 :         return TRUE;
      80             :     }
      81             : 
      82       61356 :     if (ZARRIsLikelyKerchunkJSONRef(poOpenInfo))
      83             :     {
      84          38 :         return TRUE;
      85             :     }
      86       61318 :     if (cpl::starts_with(osvFilename, JSON_REF_FS_PREFIX))
      87             :     {
      88           4 :         return -1;
      89             :     }
      90      245154 :     for (const char *pszFile :
      91      306468 :          {".zarray", ".zgroup", ".zmetadata", "zarr.json"})
      92             :     {
      93      245244 :         if (cpl::ends_with(osvFilename, pszFile))
      94             :         {
      95          90 :             return TRUE;
      96             :         }
      97             :     }
      98       61224 :     if (!poOpenInfo->bIsDirectory)
      99             :     {
     100       57827 :         return FALSE;
     101             :     }
     102             : 
     103        3397 :     return CheckExistenceOfOneZarrFile(poOpenInfo->pszFilename);
     104             : }
     105             : 
     106             : /************************************************************************/
     107             : /*                    ZARRDriverSetCommonMetadata()                     */
     108             : /************************************************************************/
     109             : 
     110        1873 : void ZARRDriverSetCommonMetadata(GDALDriver *poDriver)
     111             : {
     112        1873 :     poDriver->SetDescription(DRIVER_NAME);
     113        1873 :     poDriver->SetMetadataItem(GDAL_DCAP_RASTER, "YES");
     114        1873 :     poDriver->SetMetadataItem(GDAL_DCAP_MULTIDIM_RASTER, "YES");
     115        1873 :     poDriver->SetMetadataItem(GDAL_DMD_LONGNAME, "Zarr");
     116        1873 :     poDriver->SetMetadataItem(GDAL_DMD_EXTENSION, "zarr");
     117        1873 :     poDriver->SetMetadataItem(
     118             :         GDAL_DMD_CREATIONDATATYPES,
     119             :         "Int8 Byte Int16 UInt16 Int32 UInt32 Int64 UInt64 "
     120        1873 :         "Float16 Float32 Float64 CFLoat16 CFloat32 CFloat64");
     121        1873 :     poDriver->SetMetadataItem(GDAL_DCAP_VIRTUALIO, "YES");
     122        1873 :     poDriver->SetMetadataItem(GDAL_DMD_SUBDATASETS, "YES");
     123        1873 :     poDriver->SetMetadataItem(GDAL_DCAP_CREATE_SUBDATASETS, "YES");
     124             : 
     125        1873 :     poDriver->SetMetadataItem(
     126             :         GDAL_DMD_OPENOPTIONLIST,
     127             :         "<OpenOptionList>"
     128             :         "   <Option name='LIST_ALL_ARRAYS' type='boolean' "
     129             :         "description='Whether to list all arrays, and not only those whose "
     130             :         "dimension count is 2 or more' default='NO'/>"
     131             :         "   <Option name='USE_CONSOLIDATED_METADATA' alias='USE_ZMETADATA' "
     132             :         "type='boolean' description='Whether "
     133             :         "to use consolidated metadata' default='YES'/>"
     134             :         "   <Option name='CACHE_TILE_PRESENCE' type='boolean' "
     135             :         "description='Whether to establish an initial listing of present "
     136             :         "tiles' default='NO'/>"
     137             :         "   <Option name='CACHE_KERCHUNK_JSON' type='boolean' "
     138             :         "description='Whether to transform Kerchunk JSON reference files into "
     139             :         "Kerchunk Parquet reference files in a local cache' default='NO'/>"
     140             :         "   <Option name='MULTIBAND' type='boolean' default='YES' "
     141             :         "description='Whether to expose >= 3D arrays as GDAL multiband "
     142             :         "datasets "
     143             :         "(when using the classic 2D API)'/>"
     144             :         "   <Option name='DIM_X' type='string' description="
     145             :         "'Name or index of the X dimension (only used when MULTIBAND=YES)'/>"
     146             :         "   <Option name='DIM_Y' type='string' description="
     147             :         "'Name or index of the Y dimension (only used when MULTIBAND=YES)'/>"
     148             :         "   <Option name='LOAD_EXTRA_DIM_METADATA_DELAY' type='string' "
     149             :         "description="
     150             :         "'Maximum delay in seconds allowed to set the DIM_{dimname}_VALUE band "
     151             :         "metadata items'/>"
     152        1873 :         "</OpenOptionList>");
     153             : 
     154        1873 :     poDriver->SetMetadataItem(
     155             :         GDAL_DMD_MULTIDIM_DATASET_CREATIONOPTIONLIST,
     156             :         "<MultiDimDatasetCreationOptionList>"
     157             :         "   <Option name='FORMAT' type='string-select' default='ZARR_V3'>"
     158             :         "     <Value>ZARR_V2</Value>"
     159             :         "     <Value>ZARR_V3</Value>"
     160             :         "   </Option>"
     161             :         "   <Option name='CREATE_CONSOLIDATED_METADATA' "
     162             :         "alias='CREATE_ZMETADATA' type='boolean' "
     163             :         "description='Whether to create consolidated metadata' default='YES'/>"
     164        1873 :         "</MultiDimDatasetCreationOptionList>");
     165             : 
     166        1873 :     poDriver->pfnIdentify = ZARRDriverIdentify;
     167        1873 :     poDriver->SetMetadataItem(GDAL_DCAP_OPEN, "YES");
     168        1873 :     poDriver->SetMetadataItem(GDAL_DCAP_CREATE, "YES");
     169        1873 :     poDriver->SetMetadataItem(GDAL_DCAP_CREATECOPY, "YES");
     170        1873 :     poDriver->SetMetadataItem(GDAL_DCAP_CREATE_MULTIDIMENSIONAL, "YES");
     171             : 
     172        3746 :     poDriver->DeclareAlgorithm({"add-georeferencing-convention"});
     173             : 
     174        1873 :     poDriver->SetMetadataItem(GDAL_DCAP_UPDATE, "YES");
     175        1873 :     poDriver->SetMetadataItem(GDAL_DMD_UPDATE_ITEMS,
     176             :                               "GeoTransform SRS NoData "
     177             :                               "RasterValues "
     178        1873 :                               "DatasetMetadata BandMetadata");
     179        1873 : }
     180             : 
     181             : /************************************************************************/
     182             : /*                     DeclareDeferredZarrPlugin()                      */
     183             : /************************************************************************/
     184             : 
     185             : #ifdef PLUGIN_FILENAME
     186             : void DeclareDeferredZarrPlugin()
     187             : {
     188             :     if (GDALGetDriverByName(DRIVER_NAME) != nullptr)
     189             :     {
     190             :         return;
     191             :     }
     192             :     auto poDriver = new GDALPluginDriverProxy(PLUGIN_FILENAME);
     193             : #ifdef PLUGIN_INSTALLATION_MESSAGE
     194             :     poDriver->SetMetadataItem(GDAL_DMD_PLUGIN_INSTALLATION_MESSAGE,
     195             :                               PLUGIN_INSTALLATION_MESSAGE);
     196             : #endif
     197             :     ZARRDriverSetCommonMetadata(poDriver);
     198             :     GetGDALDriverManager()->DeclareDeferredPluginDriver(poDriver);
     199             : 
     200             :     const auto loadLambda = []()
     201             :     {
     202             :         auto poDrv = GetGDALDriverManager()->GetDriverByName(DRIVER_NAME);
     203             :         // Querying any non-standard driver metadata forces the plugin
     204             :         // to be loaded.
     205             :         if (poDrv)
     206             :             poDrv->GetMetadata("FORCE_LOAD");
     207             :     };
     208             : 
     209             :     // trigger the loading of the plugin if Kerchunk related file systems are
     210             :     // queried.
     211             :     VSIFileManager::RegisterHandlerLoader(JSON_REF_FS_PREFIX, loadLambda);
     212             :     VSIFileManager::RegisterHandlerLoader(PARQUET_REF_FS_PREFIX, loadLambda);
     213             : }
     214             : #endif