Line data    Source code

       1             : /******************************************************************************
       2             :  *
       3             :  * Project:  Feather Translator
       4             :  * Purpose:  Implements OGRFeatherDriver.
       5             :  * Author:   Even Rouault, <even.rouault at spatialys.com>
       6             :  *
       7             :  ******************************************************************************
       8             :  * Copyright (c) 2022, Planet Labs
       9             :  *
      10             :  * SPDX-License-Identifier: MIT
      11             :  ****************************************************************************/
      12             : 
      13             : #include "ogrsf_frmts.h"
      14             : #include "gdal_priv.h"
      15             : 
      16             : #include "ogrfeatherdrivercore.h"
      17             : 
      18             : /************************************************************************/
      19             : /*              OGRFeatherDriverIsArrowIPCStreamBasic()                 */
      20             : /************************************************************************/
      21             : 
      22       43624 : static int OGRFeatherDriverIsArrowIPCStreamBasic(GDALOpenInfo *poOpenInfo)
      23             : {
      24             :     // WARNING: if making changes in that method, reflect them in
      25             :     // IsArrowIPCStream() in ogrfeatherdriver.cpp
      26             : 
      27       43624 :     if (STARTS_WITH_CI(poOpenInfo->pszFilename, "ARROW_IPC_STREAM:"))
      28           3 :         return true;
      29             : 
      30       43621 :     constexpr int CONTINUATION_SIZE = 4;  // 0xFFFFFFFF
      31       43621 :     constexpr int METADATA_SIZE_SIZE = 4;
      32             : 
      33             :     // See
      34             :     // https://arrow.apache.org/docs/format/Columnar.html#encapsulated-message-format
      35       43621 :     if (poOpenInfo->fpL != nullptr &&
      36        1573 :         poOpenInfo->nHeaderBytes >= CONTINUATION_SIZE + METADATA_SIZE_SIZE &&
      37        1458 :         memcmp(poOpenInfo->pabyHeader, "\xFF\xFF\xFF\xFF", CONTINUATION_SIZE) ==
      38             :             0)
      39             :     {
      40          13 :         const char *pszExt = CPLGetExtension(poOpenInfo->pszFilename);
      41          13 :         if (EQUAL(pszExt, "arrows") || EQUAL(pszExt, "ipc"))
      42           3 :             return true;
      43             : 
      44          10 :         const uint32_t nMetadataSize =
      45          10 :             CPL_LSBUINT32PTR(poOpenInfo->pabyHeader + CONTINUATION_SIZE);
      46          10 :         if (strcmp(poOpenInfo->pszFilename, "/vsistdin/") == 0)
      47             :         {
      48             :             // Padding after metadata and before body is not necessarily present
      49             :             // but the body must be at least 4 bytes
      50           1 :             constexpr int PADDING_MAX_SIZE = 4;
      51             : 
      52             :             // /vsistdin/ cannot seek back beyond first MB
      53           1 :             if (nMetadataSize >
      54             :                 1024 * 1024 -
      55             :                     (CONTINUATION_SIZE + METADATA_SIZE_SIZE + PADDING_MAX_SIZE))
      56             :             {
      57           1 :                 if (poOpenInfo->IsSingleAllowedDriver("ARROW"))
      58           1 :                     return true;
      59           0 :                 return GDAL_IDENTIFY_UNKNOWN;
      60             :             }
      61           0 :             const int nSizeToRead = CONTINUATION_SIZE + METADATA_SIZE_SIZE +
      62           0 :                                     nMetadataSize + PADDING_MAX_SIZE;
      63           0 :             if (!poOpenInfo->TryToIngest(nSizeToRead))
      64             :             {
      65           0 :                 return false;
      66             :             }
      67             : 
      68           0 :             if (poOpenInfo->IsSingleAllowedDriver("ARROW"))
      69           0 :                 return true;
      70           0 :             return GDAL_IDENTIFY_UNKNOWN;
      71             :         }
      72             : 
      73           9 :         VSIFSeekL(poOpenInfo->fpL, 0, SEEK_END);
      74           9 :         const auto nFileSize = VSIFTellL(poOpenInfo->fpL);
      75           9 :         VSIFSeekL(poOpenInfo->fpL, 0, SEEK_SET);
      76           9 :         if (nMetadataSize >
      77           9 :             nFileSize - (CONTINUATION_SIZE + METADATA_SIZE_SIZE))
      78           4 :             return false;
      79             : 
      80           5 :         if (poOpenInfo->IsSingleAllowedDriver("ARROW"))
      81           0 :             return true;
      82           5 :         return GDAL_IDENTIFY_UNKNOWN;
      83             :     }
      84       43608 :     return false;
      85             : }
      86             : 
      87             : /************************************************************************/
      88             : /*                    OGRFeatherDriverIsArrowFileFormat()               */
      89             : /************************************************************************/
      90             : 
      91             : template <size_t N> constexpr int constexpr_length(const char (&)[N])
      92             : {
      93             :     return static_cast<int>(N - 1);
      94             : }
      95             : 
      96       44148 : bool OGRFeatherDriverIsArrowFileFormat(GDALOpenInfo *poOpenInfo)
      97             : {
      98             :     // See https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
      99       44148 :     bool bRet = false;
     100       44148 :     constexpr const char SIGNATURE[] = "ARROW1";
     101       44148 :     constexpr int SIGNATURE_SIZE = constexpr_length(SIGNATURE);
     102             :     static_assert(SIGNATURE_SIZE == 6, "SIGNATURE_SIZE == 6");
     103       44148 :     constexpr int SIGNATURE_PLUS_PADDING = SIGNATURE_SIZE + 2;
     104       44148 :     constexpr int FOOTERSIZE_SIZE = 4;
     105       44148 :     if (poOpenInfo->fpL != nullptr &&
     106        2100 :         poOpenInfo->nHeaderBytes >=
     107        1954 :             SIGNATURE_PLUS_PADDING + FOOTERSIZE_SIZE + SIGNATURE_SIZE &&
     108        1954 :         memcmp(poOpenInfo->pabyHeader, SIGNATURE, SIGNATURE_SIZE) == 0)
     109             :     {
     110        1072 :         VSIFSeekL(poOpenInfo->fpL, 0, SEEK_END);
     111        1072 :         const auto nFileSize = VSIFTellL(poOpenInfo->fpL);
     112        1072 :         VSIFSeekL(poOpenInfo->fpL,
     113             :                   nFileSize - (FOOTERSIZE_SIZE + SIGNATURE_SIZE), SEEK_SET);
     114        1072 :         uint32_t nFooterSize = 0;
     115             :         static_assert(sizeof(nFooterSize) == FOOTERSIZE_SIZE,
     116             :                       "sizeof(nFooterSize) == FOOTERSIZE_SIZE");
     117        1072 :         VSIFReadL(&nFooterSize, 1, sizeof(nFooterSize), poOpenInfo->fpL);
     118        1072 :         CPL_LSBPTR32(&nFooterSize);
     119        1072 :         unsigned char abyTrailingBytes[SIGNATURE_SIZE] = {0};
     120        1072 :         VSIFReadL(&abyTrailingBytes[0], 1, SIGNATURE_SIZE, poOpenInfo->fpL);
     121        2144 :         bRet = memcmp(abyTrailingBytes, SIGNATURE, SIGNATURE_SIZE) == 0 &&
     122        1072 :                nFooterSize < nFileSize;
     123        1072 :         VSIFSeekL(poOpenInfo->fpL, 0, SEEK_SET);
     124             :     }
     125       44148 :     return bRet;
     126             : }
     127             : 
     128             : /************************************************************************/
     129             : /*                             Identify()                               */
     130             : /************************************************************************/
     131             : 
     132       43625 : int OGRFeatherDriverIdentify(GDALOpenInfo *poOpenInfo)
     133             : {
     134       43625 :     if (STARTS_WITH(poOpenInfo->pszFilename, "gdalvsi://"))
     135             :     {
     136           1 :         GDALOpenInfo oOpenInfo(poOpenInfo->pszFilename + strlen("gdalvsi://"),
     137           2 :                                poOpenInfo->nOpenFlags);
     138           1 :         return OGRFeatherDriverIdentify(&oOpenInfo);
     139             :     }
     140             : 
     141       43624 :     int ret = OGRFeatherDriverIsArrowIPCStreamBasic(poOpenInfo);
     142       43624 :     if (ret == GDAL_IDENTIFY_TRUE || ret == GDAL_IDENTIFY_UNKNOWN)
     143          12 :         return ret;
     144       43612 :     return OGRFeatherDriverIsArrowFileFormat(poOpenInfo);
     145             : }
     146             : 
     147             : /************************************************************************/
     148             : /*                OGRFeatherDriverSetCommonMetadata()                   */
     149             : /************************************************************************/
     150             : 
     151        1306 : void OGRFeatherDriverSetCommonMetadata(GDALDriver *poDriver)
     152             : {
     153        1306 :     poDriver->SetDescription(DRIVER_NAME);
     154        1306 :     poDriver->SetMetadataItem(GDAL_DCAP_VECTOR, "YES");
     155        1306 :     poDriver->SetMetadataItem(GDAL_DCAP_CREATE_LAYER, "YES");
     156        1306 :     poDriver->SetMetadataItem(GDAL_DCAP_CREATE_FIELD, "YES");
     157        1306 :     poDriver->SetMetadataItem(GDAL_DMD_LONGNAME,
     158        1306 :                               "(Geo)Arrow IPC File Format / Stream");
     159        1306 :     poDriver->SetMetadataItem(GDAL_DMD_EXTENSIONS, "arrow feather arrows ipc");
     160        1306 :     poDriver->SetMetadataItem(GDAL_DMD_HELPTOPIC,
     161        1306 :                               "drivers/vector/feather.html");
     162        1306 :     poDriver->SetMetadataItem(GDAL_DCAP_VIRTUALIO, "YES");
     163        1306 :     poDriver->SetMetadataItem(GDAL_DCAP_MEASURED_GEOMETRIES, "YES");
     164        1306 :     poDriver->SetMetadataItem(GDAL_DCAP_Z_GEOMETRIES, "YES");
     165        1306 :     poDriver->SetMetadataItem(GDAL_DMD_SUPPORTED_SQL_DIALECTS, "OGRSQL SQLITE");
     166             : 
     167        1306 :     poDriver->SetMetadataItem(
     168             :         GDAL_DMD_CREATIONFIELDDATATYPES,
     169             :         "Integer Integer64 Real String Date Time DateTime "
     170        1306 :         "Binary IntegerList Integer64List RealList StringList");
     171        1306 :     poDriver->SetMetadataItem(GDAL_DMD_CREATIONFIELDDATASUBTYPES,
     172        1306 :                               "Boolean Int16 Float32 JSON UUID");
     173        1306 :     poDriver->SetMetadataItem(GDAL_DMD_CREATION_FIELD_DEFN_FLAGS,
     174             :                               "WidthPrecision Nullable "
     175        1306 :                               "Comment AlternativeName Domain");
     176             : 
     177        1306 :     poDriver->pfnIdentify = OGRFeatherDriverIdentify;
     178        1306 :     poDriver->SetMetadataItem(GDAL_DCAP_OPEN, "YES");
     179        1306 :     poDriver->SetMetadataItem(GDAL_DCAP_CREATE, "YES");
     180        1306 : }
     181             : 
     182             : /************************************************************************/
     183             : /*                  DeclareDeferredOGRArrowPlugin()                     */
     184             : /************************************************************************/
     185             : 
     186             : #ifdef PLUGIN_FILENAME
     187        1595 : void DeclareDeferredOGRArrowPlugin()
     188             : {
     189        1595 :     if (GDALGetDriverByName(DRIVER_NAME) != nullptr)
     190             :     {
     191         302 :         return;
     192             :     }
     193        1293 :     auto poDriver = new GDALPluginDriverProxy(PLUGIN_FILENAME);
     194             : #ifdef PLUGIN_INSTALLATION_MESSAGE
     195             :     poDriver->SetMetadataItem(GDAL_DMD_PLUGIN_INSTALLATION_MESSAGE,
     196             :                               PLUGIN_INSTALLATION_MESSAGE);
     197             : #endif
     198        1293 :     OGRFeatherDriverSetCommonMetadata(poDriver);
     199        1293 :     GetGDALDriverManager()->DeclareDeferredPluginDriver(poDriver);
     200             : }
     201             : #endif