Line data    Source code

       1             : /******************************************************************************
       2             :  *
       3             :  * Project:  Parquet Translator
       4             :  * Purpose:  Implements OGRParquetDriver.
       5             :  * Author:   Even Rouault, <even.rouault at spatialys.com>
       6             :  *
       7             :  ******************************************************************************
       8             :  * Copyright (c) 2022, Planet Labs
       9             :  *
      10             :  * SPDX-License-Identifier: MIT
      11             :  ****************************************************************************/
      12             : 
      13             : #include "gdal_frmts.h"
      14             : 
      15             : #ifdef PLUGIN_FILENAME
      16             : #include "gdalplugindriverproxy.h"
      17             : #endif
      18             : 
      19             : #include "ogrsf_frmts.h"
      20             : #include "gdal_priv.h"
      21             : 
      22             : #include "ogrparquetdrivercore.h"
      23             : 
      24             : /************************************************************************/
      25             : /*                              Identify()                              */
      26             : /************************************************************************/
      27             : 
      28             : template <size_t N> constexpr int constexpr_length(const char (&)[N])
      29             : {
      30             :     return static_cast<int>(N - 1);
      31             : }
      32             : 
      33       55674 : int OGRParquetDriverIdentify(GDALOpenInfo *poOpenInfo)
      34             : {
      35             : #if defined(GDAL_USE_ARROWDATASET) || defined(PLUGIN_FILENAME)
      36       55674 :     if (poOpenInfo->bIsDirectory)
      37             :     {
      38             :         // Might be a ParquetDataset
      39        1330 :         return -1;
      40             :     }
      41             : #endif
      42       54344 :     if (STARTS_WITH(poOpenInfo->pszFilename, "PARQUET:"))
      43         345 :         return TRUE;
      44             : 
      45             :     // See https://github.com/apache/parquet-format#file-format
      46       53999 :     bool bRet = false;
      47       53999 :     constexpr const char SIGNATURE[] = "PAR1";
      48       53999 :     constexpr int SIGNATURE_SIZE = constexpr_length(SIGNATURE);
      49             :     static_assert(SIGNATURE_SIZE == 4, "SIGNATURE_SIZE == 4");
      50       53999 :     constexpr int METADATASIZE_SIZE = 4;
      51       53999 :     if (poOpenInfo->fpL != nullptr &&
      52        4152 :         poOpenInfo->nHeaderBytes >=
      53        4012 :             SIGNATURE_SIZE + METADATASIZE_SIZE + SIGNATURE_SIZE &&
      54        4012 :         memcmp(poOpenInfo->pabyHeader, SIGNATURE, SIGNATURE_SIZE) == 0)
      55             :     {
      56        2112 :         VSIFSeekL(poOpenInfo->fpL, 0, SEEK_END);
      57        2112 :         const auto nFileSize = VSIFTellL(poOpenInfo->fpL);
      58        2112 :         VSIFSeekL(poOpenInfo->fpL,
      59             :                   nFileSize - (METADATASIZE_SIZE + SIGNATURE_SIZE), SEEK_SET);
      60        2112 :         uint32_t nMetadataSize = 0;
      61             :         static_assert(sizeof(nMetadataSize) == METADATASIZE_SIZE,
      62             :                       "sizeof(nMetadataSize) == METADATASIZE_SIZE");
      63        2112 :         VSIFReadL(&nMetadataSize, 1, sizeof(nMetadataSize), poOpenInfo->fpL);
      64        2112 :         CPL_LSBPTR32(&nMetadataSize);
      65        2112 :         unsigned char abyTrailingBytes[SIGNATURE_SIZE] = {0};
      66        2112 :         VSIFReadL(&abyTrailingBytes[0], 1, SIGNATURE_SIZE, poOpenInfo->fpL);
      67        4224 :         bRet = memcmp(abyTrailingBytes, SIGNATURE, SIGNATURE_SIZE) == 0 &&
      68        2112 :                nMetadataSize < nFileSize;
      69        2112 :         VSIFSeekL(poOpenInfo->fpL, 0, SEEK_SET);
      70             :     }
      71       53999 :     return bRet;
      72             : }
      73             : 
      74             : /************************************************************************/
      75             : /*                 OGRParquetDriverSetCommonMetadata()                  */
      76             : /************************************************************************/
      77             : 
      78        1864 : void OGRParquetDriverSetCommonMetadata(GDALDriver *poDriver)
      79             : {
      80        1864 :     poDriver->SetDescription(DRIVER_NAME);
      81        1864 :     poDriver->SetMetadataItem(GDAL_DCAP_VECTOR, "YES");
      82        1864 :     poDriver->SetMetadataItem(GDAL_DCAP_CREATE_LAYER, "YES");
      83        1864 :     poDriver->SetMetadataItem(GDAL_DMD_LONGNAME, "(Geo)Parquet");
      84        1864 :     poDriver->SetMetadataItem(GDAL_DMD_EXTENSION, "parquet");
      85        1864 :     poDriver->SetMetadataItem(GDAL_DMD_HELPTOPIC,
      86        1864 :                               "drivers/vector/parquet.html");
      87        1864 :     poDriver->SetMetadataItem(GDAL_DCAP_VIRTUALIO, "YES");
      88        1864 :     poDriver->SetMetadataItem(GDAL_DCAP_MEASURED_GEOMETRIES, "YES");
      89        1864 :     poDriver->SetMetadataItem(GDAL_DCAP_Z_GEOMETRIES, "YES");
      90        1864 :     poDriver->SetMetadataItem(GDAL_DCAP_REOPEN_AFTER_WRITE_REQUIRED, "YES");
      91        1864 :     poDriver->SetMetadataItem(GDAL_DCAP_CAN_READ_AFTER_DELETE, "YES");
      92             : 
      93        1864 :     poDriver->SetMetadataItem(GDAL_DCAP_CREATE_FIELD, "YES");
      94        1864 :     poDriver->SetMetadataItem(
      95             :         GDAL_DMD_CREATIONFIELDDATATYPES,
      96             :         "Integer Integer64 Real String Date Time DateTime "
      97        1864 :         "Binary IntegerList Integer64List RealList StringList");
      98        1864 :     poDriver->SetMetadataItem(GDAL_DMD_CREATIONFIELDDATASUBTYPES,
      99        1864 :                               "Boolean Int16 Float32 JSON UUID");
     100        1864 :     poDriver->SetMetadataItem(GDAL_DMD_CREATION_FIELD_DEFN_FLAGS,
     101             :                               "WidthPrecision Nullable Comment "
     102        1864 :                               "AlternativeName Domain");
     103        1864 :     poDriver->SetMetadataItem(GDAL_DMD_SUPPORTED_SQL_DIALECTS, "OGRSQL SQLITE");
     104             : 
     105        1864 :     poDriver->SetMetadataItem(GDAL_DMD_CREATIONOPTIONLIST,
     106             :                               "<CreationOptionList>"
     107        1864 :                               "</CreationOptionList>");
     108             : 
     109        1864 :     poDriver->SetMetadataItem(
     110             :         GDAL_DMD_OPENOPTIONLIST,
     111             :         "<OpenOptionList>"
     112             :         "  <Option name='GEOM_POSSIBLE_NAMES' type='string' description='Comma "
     113             :         "separated list of possible names for geometry column(s).' "
     114             :         "default='geometry,wkb_geometry,wkt_geometry'/>"
     115             :         "  <Option name='CRS' type='string' "
     116             :         "description='Set/override CRS, typically defined as AUTH:CODE "
     117             :         "(e.g EPSG:4326), of geometry column(s)'/>"
     118             :         "  <Option name='LISTS_AS_STRING_JSON' type='boolean' description='"
     119             :         "Whether lists of strings/integers/reals should be reported as "
     120             :         "String(JSON) fields rather than String/Integer[64]/RealList. Useful "
     121             :         "when null values in such lists must be exactly mapped as such.' "
     122             :         "default='NO'/>"
     123        1864 :         "</OpenOptionList>");
     124             : 
     125        1864 :     poDriver->pfnIdentify = OGRParquetDriverIdentify;
     126        1864 :     poDriver->SetMetadataItem(GDAL_DCAP_OPEN, "YES");
     127        1864 :     poDriver->SetMetadataItem(GDAL_DCAP_CREATE, "YES");
     128             : 
     129        1864 :     poDriver->SetMetadataItem(GDAL_DCAP_UPDATE, "YES");
     130        1864 :     poDriver->SetMetadataItem(GDAL_DMD_UPDATE_ITEMS, "Features");
     131             : 
     132        1864 :     poDriver->SetMetadataItem(GDAL_DCAP_DELETE_FIELD, "YES");
     133        1864 :     poDriver->SetMetadataItem(GDAL_DCAP_REORDER_FIELDS, "YES");
     134        1864 :     poDriver->SetMetadataItem(GDAL_DMD_ALTER_FIELD_DEFN_FLAGS,
     135        1864 :                               "Name Type WidthPrecision");
     136             : 
     137        3728 :     poDriver->DeclareAlgorithm({"create-metadata-file"});
     138        1864 : }
     139             : 
     140             : /************************************************************************/
     141             : /*                  DeclareDeferredOGRParquetPlugin()                   */
     142             : /************************************************************************/
     143             : 
     144             : #ifdef PLUGIN_FILENAME
     145        2055 : void DeclareDeferredOGRParquetPlugin()
     146             : {
     147        2055 :     if (GDALGetDriverByName(DRIVER_NAME) != nullptr)
     148             :     {
     149         263 :         return;
     150             :     }
     151        1792 :     auto poDriver = new GDALPluginDriverProxy(PLUGIN_FILENAME);
     152             : #ifdef PLUGIN_INSTALLATION_MESSAGE
     153             :     poDriver->SetMetadataItem(GDAL_DMD_PLUGIN_INSTALLATION_MESSAGE,
     154             :                               PLUGIN_INSTALLATION_MESSAGE);
     155             : #endif
     156        1792 :     OGRParquetDriverSetCommonMetadata(poDriver);
     157        1792 :     GetGDALDriverManager()->DeclareDeferredPluginDriver(poDriver);
     158             : }
     159             : #endif