Line data    Source code

       1             : /******************************************************************************
       2             :  *
       3             :  * Project:  Feather Translator
       4             :  * Purpose:  Implements OGRFeatherDriver.
       5             :  * Author:   Even Rouault, <even.rouault at spatialys.com>
       6             :  *
       7             :  ******************************************************************************
       8             :  * Copyright (c) 2022, Planet Labs
       9             :  *
      10             :  * SPDX-License-Identifier: MIT
      11             :  ****************************************************************************/
      12             : 
      13             : #include "gdal_frmts.h"
      14             : 
      15             : #ifdef PLUGIN_FILENAME
      16             : #include "gdalplugindriverproxy.h"
      17             : #endif
      18             : 
      19             : #include "ogrsf_frmts.h"
      20             : #include "gdal_priv.h"
      21             : 
      22             : #include "ogrfeatherdrivercore.h"
      23             : 
      24             : /************************************************************************/
      25             : /*              OGRFeatherDriverIsArrowIPCStreamBasic()                 */
      26             : /************************************************************************/
      27             : 
      28       51499 : static int OGRFeatherDriverIsArrowIPCStreamBasic(GDALOpenInfo *poOpenInfo)
      29             : {
      30             :     // WARNING: if making changes in that method, reflect them in
      31             :     // IsArrowIPCStream() in ogrfeatherdriver.cpp
      32             : 
      33       51499 :     if (STARTS_WITH_CI(poOpenInfo->pszFilename, "ARROW_IPC_STREAM:"))
      34           3 :         return true;
      35             : 
      36       51496 :     constexpr int CONTINUATION_SIZE = 4;  // 0xFFFFFFFF
      37       51496 :     constexpr int METADATA_SIZE_SIZE = 4;
      38             : 
      39             :     // See
      40             :     // https://arrow.apache.org/docs/format/Columnar.html#encapsulated-message-format
      41       51496 :     if (poOpenInfo->fpL != nullptr &&
      42        1985 :         poOpenInfo->nHeaderBytes >= CONTINUATION_SIZE + METADATA_SIZE_SIZE &&
      43        1856 :         memcmp(poOpenInfo->pabyHeader, "\xFF\xFF\xFF\xFF", CONTINUATION_SIZE) ==
      44             :             0)
      45             :     {
      46          13 :         const char *pszExt = poOpenInfo->osExtension.c_str();
      47          13 :         if (EQUAL(pszExt, "arrows") || EQUAL(pszExt, "ipc"))
      48           3 :             return true;
      49             : 
      50          10 :         const uint32_t nMetadataSize =
      51          10 :             CPL_LSBUINT32PTR(poOpenInfo->pabyHeader + CONTINUATION_SIZE);
      52          10 :         if (strcmp(poOpenInfo->pszFilename, "/vsistdin/") == 0)
      53             :         {
      54             :             // Padding after metadata and before body is not necessarily present
      55             :             // but the body must be at least 4 bytes
      56           1 :             constexpr int PADDING_MAX_SIZE = 4;
      57             : 
      58             :             // /vsistdin/ cannot seek back beyond first MB
      59           1 :             if (nMetadataSize >
      60             :                 1024 * 1024 -
      61             :                     (CONTINUATION_SIZE + METADATA_SIZE_SIZE + PADDING_MAX_SIZE))
      62             :             {
      63           1 :                 if (poOpenInfo->IsSingleAllowedDriver("ARROW"))
      64           1 :                     return true;
      65           0 :                 return GDAL_IDENTIFY_UNKNOWN;
      66             :             }
      67           0 :             const int nSizeToRead = CONTINUATION_SIZE + METADATA_SIZE_SIZE +
      68           0 :                                     nMetadataSize + PADDING_MAX_SIZE;
      69           0 :             if (!poOpenInfo->TryToIngest(nSizeToRead))
      70             :             {
      71           0 :                 return false;
      72             :             }
      73             : 
      74           0 :             if (poOpenInfo->IsSingleAllowedDriver("ARROW"))
      75           0 :                 return true;
      76           0 :             return GDAL_IDENTIFY_UNKNOWN;
      77             :         }
      78             : 
      79           9 :         VSIFSeekL(poOpenInfo->fpL, 0, SEEK_END);
      80           9 :         const auto nFileSize = VSIFTellL(poOpenInfo->fpL);
      81           9 :         VSIFSeekL(poOpenInfo->fpL, 0, SEEK_SET);
      82           9 :         if (nMetadataSize >
      83           9 :             nFileSize - (CONTINUATION_SIZE + METADATA_SIZE_SIZE))
      84           4 :             return false;
      85             : 
      86           5 :         if (poOpenInfo->IsSingleAllowedDriver("ARROW"))
      87           0 :             return true;
      88           5 :         return GDAL_IDENTIFY_UNKNOWN;
      89             :     }
      90       51483 :     return false;
      91             : }
      92             : 
      93             : /************************************************************************/
      94             : /*                    OGRFeatherDriverIsArrowFileFormat()               */
      95             : /************************************************************************/
      96             : 
      97             : template <size_t N> constexpr int constexpr_length(const char (&)[N])
      98             : {
      99             :     return static_cast<int>(N - 1);
     100             : }
     101             : 
     102       52023 : bool OGRFeatherDriverIsArrowFileFormat(GDALOpenInfo *poOpenInfo)
     103             : {
     104             :     // See https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
     105       52023 :     bool bRet = false;
     106       52023 :     constexpr const char SIGNATURE[] = "ARROW1";
     107       52023 :     constexpr int SIGNATURE_SIZE = constexpr_length(SIGNATURE);
     108             :     static_assert(SIGNATURE_SIZE == 6, "SIGNATURE_SIZE == 6");
     109       52023 :     constexpr int SIGNATURE_PLUS_PADDING = SIGNATURE_SIZE + 2;
     110       52023 :     constexpr int FOOTERSIZE_SIZE = 4;
     111       52023 :     if (poOpenInfo->fpL != nullptr &&
     112        2512 :         poOpenInfo->nHeaderBytes >=
     113        2353 :             SIGNATURE_PLUS_PADDING + FOOTERSIZE_SIZE + SIGNATURE_SIZE &&
     114        2353 :         memcmp(poOpenInfo->pabyHeader, SIGNATURE, SIGNATURE_SIZE) == 0)
     115             :     {
     116        1072 :         VSIFSeekL(poOpenInfo->fpL, 0, SEEK_END);
     117        1072 :         const auto nFileSize = VSIFTellL(poOpenInfo->fpL);
     118        1072 :         VSIFSeekL(poOpenInfo->fpL,
     119             :                   nFileSize - (FOOTERSIZE_SIZE + SIGNATURE_SIZE), SEEK_SET);
     120        1072 :         uint32_t nFooterSize = 0;
     121             :         static_assert(sizeof(nFooterSize) == FOOTERSIZE_SIZE,
     122             :                       "sizeof(nFooterSize) == FOOTERSIZE_SIZE");
     123        1072 :         VSIFReadL(&nFooterSize, 1, sizeof(nFooterSize), poOpenInfo->fpL);
     124        1072 :         CPL_LSBPTR32(&nFooterSize);
     125        1072 :         unsigned char abyTrailingBytes[SIGNATURE_SIZE] = {0};
     126        1072 :         VSIFReadL(&abyTrailingBytes[0], 1, SIGNATURE_SIZE, poOpenInfo->fpL);
     127        2144 :         bRet = memcmp(abyTrailingBytes, SIGNATURE, SIGNATURE_SIZE) == 0 &&
     128        1072 :                nFooterSize < nFileSize;
     129        1072 :         VSIFSeekL(poOpenInfo->fpL, 0, SEEK_SET);
     130             :     }
     131       52023 :     return bRet;
     132             : }
     133             : 
     134             : /************************************************************************/
     135             : /*                             Identify()                               */
     136             : /************************************************************************/
     137             : 
     138       51500 : int OGRFeatherDriverIdentify(GDALOpenInfo *poOpenInfo)
     139             : {
     140       51500 :     if (STARTS_WITH(poOpenInfo->pszFilename, "gdalvsi://"))
     141             :     {
     142           1 :         GDALOpenInfo oOpenInfo(poOpenInfo->pszFilename + strlen("gdalvsi://"),
     143           2 :                                poOpenInfo->nOpenFlags);
     144           1 :         return OGRFeatherDriverIdentify(&oOpenInfo);
     145             :     }
     146             : 
     147       51499 :     int ret = OGRFeatherDriverIsArrowIPCStreamBasic(poOpenInfo);
     148       51499 :     if (ret == GDAL_IDENTIFY_TRUE || ret == GDAL_IDENTIFY_UNKNOWN)
     149          12 :         return ret;
     150       51487 :     return OGRFeatherDriverIsArrowFileFormat(poOpenInfo);
     151             : }
     152             : 
     153             : /************************************************************************/
     154             : /*                OGRFeatherDriverSetCommonMetadata()                   */
     155             : /************************************************************************/
     156             : 
     157        1769 : void OGRFeatherDriverSetCommonMetadata(GDALDriver *poDriver)
     158             : {
     159        1769 :     poDriver->SetDescription(DRIVER_NAME);
     160        1769 :     poDriver->SetMetadataItem(GDAL_DCAP_VECTOR, "YES");
     161        1769 :     poDriver->SetMetadataItem(GDAL_DCAP_CREATE_LAYER, "YES");
     162        1769 :     poDriver->SetMetadataItem(GDAL_DCAP_CREATE_FIELD, "YES");
     163        1769 :     poDriver->SetMetadataItem(GDAL_DMD_LONGNAME,
     164        1769 :                               "(Geo)Arrow IPC File Format / Stream");
     165        1769 :     poDriver->SetMetadataItem(GDAL_DMD_EXTENSIONS, "arrow feather arrows ipc");
     166        1769 :     poDriver->SetMetadataItem(GDAL_DMD_HELPTOPIC,
     167        1769 :                               "drivers/vector/feather.html");
     168        1769 :     poDriver->SetMetadataItem(GDAL_DCAP_VIRTUALIO, "YES");
     169        1769 :     poDriver->SetMetadataItem(GDAL_DCAP_MEASURED_GEOMETRIES, "YES");
     170        1769 :     poDriver->SetMetadataItem(GDAL_DCAP_Z_GEOMETRIES, "YES");
     171        1769 :     poDriver->SetMetadataItem(GDAL_DMD_SUPPORTED_SQL_DIALECTS, "OGRSQL SQLITE");
     172        1769 :     poDriver->SetMetadataItem(GDAL_DCAP_REOPEN_AFTER_WRITE_REQUIRED, "YES");
     173        1769 :     poDriver->SetMetadataItem(GDAL_DCAP_CAN_READ_AFTER_DELETE, "YES");
     174             : 
     175        1769 :     poDriver->SetMetadataItem(
     176             :         GDAL_DMD_CREATIONFIELDDATATYPES,
     177             :         "Integer Integer64 Real String Date Time DateTime "
     178        1769 :         "Binary IntegerList Integer64List RealList StringList");
     179        1769 :     poDriver->SetMetadataItem(GDAL_DMD_CREATIONFIELDDATASUBTYPES,
     180        1769 :                               "Boolean Int16 Float32 JSON UUID");
     181        1769 :     poDriver->SetMetadataItem(GDAL_DMD_CREATION_FIELD_DEFN_FLAGS,
     182             :                               "WidthPrecision Nullable "
     183        1769 :                               "Comment AlternativeName Domain");
     184             : 
     185        1769 :     poDriver->pfnIdentify = OGRFeatherDriverIdentify;
     186        1769 :     poDriver->SetMetadataItem(GDAL_DCAP_OPEN, "YES");
     187        1769 :     poDriver->SetMetadataItem(GDAL_DCAP_CREATE, "YES");
     188        1769 : }
     189             : 
     190             : /************************************************************************/
     191             : /*                  DeclareDeferredOGRArrowPlugin()                     */
     192             : /************************************************************************/
     193             : 
     194             : #ifdef PLUGIN_FILENAME
     195        2038 : void DeclareDeferredOGRArrowPlugin()
     196             : {
     197        2038 :     if (GDALGetDriverByName(DRIVER_NAME) != nullptr)
     198             :     {
     199         283 :         return;
     200             :     }
     201        1755 :     auto poDriver = new GDALPluginDriverProxy(PLUGIN_FILENAME);
     202             : #ifdef PLUGIN_INSTALLATION_MESSAGE
     203             :     poDriver->SetMetadataItem(GDAL_DMD_PLUGIN_INSTALLATION_MESSAGE,
     204             :                               PLUGIN_INSTALLATION_MESSAGE);
     205             : #endif
     206        1755 :     OGRFeatherDriverSetCommonMetadata(poDriver);
     207        1755 :     GetGDALDriverManager()->DeclareDeferredPluginDriver(poDriver);
     208             : }
     209             : #endif