Line data    Source code

       1             : /******************************************************************************
       2             :  *
       3             :  * Project:  Parquet Translator
       4             :  * Purpose:  Implements OGRParquetDriver.
       5             :  * Author:   Even Rouault, <even.rouault at spatialys.com>
       6             :  *
       7             :  ******************************************************************************
       8             :  * Copyright (c) 2022, Planet Labs
       9             :  *
      10             :  * SPDX-License-Identifier: MIT
      11             :  ****************************************************************************/
      12             : 
      13             : #ifdef STANDALONE
      14             : #include "gdal_version.h"
      15             : #else
      16             : #undef DO_NOT_DEFINE_GDAL_DATE_NAME
      17             : #include "gdal_version_full/gdal_version.h"
      18             : #endif
      19             : 
      20             : #include "ogr_parquet.h"
      21             : 
      22             : #include "../arrow_common/ograrrowwriterlayer.hpp"
      23             : 
      24             : #include "ogr_wkb.h"
      25             : 
      26             : #include <cassert>
      27             : #include <utility>
      28             : 
      29             : /************************************************************************/
      30             : /*                       OGRParquetWriterLayer()                        */
      31             : /************************************************************************/
      32             : 
      33         364 : OGRParquetWriterLayer::OGRParquetWriterLayer(
      34             :     OGRParquetWriterDataset *poDataset, arrow::MemoryPool *poMemoryPool,
      35             :     const std::shared_ptr<arrow::io::OutputStream> &poOutputStream,
      36         364 :     const char *pszLayerName)
      37             :     : OGRArrowWriterLayer(poMemoryPool, poOutputStream, pszLayerName),
      38         364 :       m_poDataset(poDataset)
      39             : {
      40         364 :     m_bWriteFieldArrowExtensionName = CPLTestBool(
      41             :         CPLGetConfigOption("OGR_PARQUET_WRITE_ARROW_EXTENSION_NAME", "NO"));
      42         364 : }
      43             : 
      44             : /************************************************************************/
      45             : /*                               Close()                                */
      46             : /************************************************************************/
      47             : 
      48         361 : bool OGRParquetWriterLayer::Close()
      49             : {
      50         361 :     if (m_poTmpGPKGLayer)
      51             :     {
      52           3 :         if (!CopyTmpGpkgLayerToFinalFile())
      53           0 :             return false;
      54             :     }
      55             : 
      56         361 :     if (m_bInitializationOK)
      57             :     {
      58         361 :         if (!FinalizeWriting())
      59           0 :             return false;
      60             :     }
      61             : 
      62         361 :     return true;
      63             : }
      64             : 
      65             : /************************************************************************/
      66             : /*                    CopyTmpGpkgLayerToFinalFile()                     */
      67             : /************************************************************************/
      68             : 
      69           3 : bool OGRParquetWriterLayer::CopyTmpGpkgLayerToFinalFile()
      70             : {
      71           3 :     if (!m_poTmpGPKGLayer)
      72             :     {
      73           0 :         return true;
      74             :     }
      75             : 
      76           3 :     CPLDebug("PARQUET", "CopyTmpGpkgLayerToFinalFile(): start...");
      77             : 
      78           3 :     VSIUnlink(m_poTmpGPKG->GetDescription());
      79             : 
      80           6 :     OGRFeature oFeat(m_poFeatureDefn);
      81             : 
      82             :     // Interval in terms of features between 2 debug progress report messages
      83           3 :     constexpr int PROGRESS_FC_INTERVAL = 100 * 1000;
      84             : 
      85             :     // First, write features without geometries
      86             :     {
      87           3 :         auto poTmpLayer = std::unique_ptr<OGRLayer>(m_poTmpGPKG->ExecuteSQL(
      88             :             "SELECT serialized_feature FROM tmp WHERE fid NOT IN (SELECT id "
      89             :             "FROM rtree_tmp_geom)",
      90           3 :             nullptr, nullptr));
      91           3 :         if (!poTmpLayer)
      92           0 :             return false;
      93        1005 :         for (const auto &poSrcFeature : poTmpLayer.get())
      94             :         {
      95        1002 :             int nBytesFeature = 0;
      96             :             const GByte *pabyFeatureData =
      97        1002 :                 poSrcFeature->GetFieldAsBinary(0, &nBytesFeature);
      98        1002 :             if (!oFeat.DeserializeFromBinary(pabyFeatureData, nBytesFeature))
      99             :             {
     100           0 :                 CPLError(CE_Failure, CPLE_AppDefined,
     101             :                          "Cannot deserialize feature");
     102           0 :                 return false;
     103             :             }
     104        1002 :             if (OGRArrowWriterLayer::ICreateFeature(&oFeat) != OGRERR_NONE)
     105             :             {
     106           0 :                 return false;
     107             :             }
     108             : 
     109        1002 :             if ((m_nFeatureCount % PROGRESS_FC_INTERVAL) == 0)
     110             :             {
     111           0 :                 CPLDebugProgress(
     112             :                     "PARQUET",
     113             :                     "CopyTmpGpkgLayerToFinalFile(): %.02f%% progress",
     114           0 :                     100.0 * double(m_nFeatureCount) /
     115           0 :                         double(m_nTmpFeatureCount));
     116             :             }
     117             :         }
     118             : 
     119           3 :         if (!FlushFeatures())
     120             :         {
     121           0 :             return false;
     122             :         }
     123             :     }
     124             : 
     125             :     // Now walk through the GPKG RTree for features with geometries
     126             :     // Cf https://github.com/sqlite/sqlite/blob/master/ext/rtree/rtree.c
     127             :     // for the description of the content of the rtree _node table
     128           6 :     std::vector<std::pair<int64_t, int>> aNodeNoDepthPair;
     129           3 :     int nTreeDepth = 0;
     130             :     // Queue the root node
     131             :     aNodeNoDepthPair.emplace_back(
     132           3 :         std::make_pair(/* nodeNo = */ 1, /* depth = */ 0));
     133           3 :     int nCountWrittenFeaturesSinceLastFlush = 0;
     134          50 :     while (!aNodeNoDepthPair.empty())
     135             :     {
     136          47 :         const auto &oLastPair = aNodeNoDepthPair.back();
     137          47 :         const int64_t nNodeNo = oLastPair.first;
     138          47 :         const int nCurDepth = oLastPair.second;
     139             :         //CPLDebug("PARQUET", "Reading nodeNode=%d, curDepth=%d", int(nNodeNo), nCurDepth);
     140          47 :         aNodeNoDepthPair.pop_back();
     141             : 
     142          47 :         auto poRTreeLayer = std::unique_ptr<OGRLayer>(m_poTmpGPKG->ExecuteSQL(
     143             :             CPLSPrintf("SELECT data FROM rtree_tmp_geom_node WHERE nodeno "
     144             :                        "= " CPL_FRMT_GIB,
     145             :                        static_cast<GIntBig>(nNodeNo)),
     146          47 :             nullptr, nullptr));
     147          47 :         if (!poRTreeLayer)
     148             :         {
     149           0 :             CPLError(CE_Failure, CPLE_AppDefined,
     150             :                      "Cannot read node " CPL_FRMT_GIB,
     151             :                      static_cast<GIntBig>(nNodeNo));
     152           0 :             return false;
     153             :         }
     154             :         const auto poRTreeFeature =
     155          47 :             std::unique_ptr<const OGRFeature>(poRTreeLayer->GetNextFeature());
     156          47 :         if (!poRTreeFeature)
     157             :         {
     158           0 :             CPLError(CE_Failure, CPLE_AppDefined,
     159             :                      "Cannot read node " CPL_FRMT_GIB,
     160             :                      static_cast<GIntBig>(nNodeNo));
     161           0 :             return false;
     162             :         }
     163             : 
     164          47 :         int nNodeBytes = 0;
     165             :         const GByte *pabyNodeData =
     166          47 :             poRTreeFeature->GetFieldAsBinary(0, &nNodeBytes);
     167          47 :         constexpr int BLOB_HEADER_SIZE = 4;
     168          47 :         if (nNodeBytes < BLOB_HEADER_SIZE)
     169             :         {
     170           0 :             CPLError(CE_Failure, CPLE_AppDefined,
     171             :                      "Not enough bytes when reading node " CPL_FRMT_GIB,
     172             :                      static_cast<GIntBig>(nNodeNo));
     173           0 :             return false;
     174             :         }
     175          47 :         if (nNodeNo == 1)
     176             :         {
     177             :             // Get the RTree depth from the root node
     178           3 :             nTreeDepth = (pabyNodeData[0] << 8) | pabyNodeData[1];
     179             :             //CPLDebug("PARQUET", "nTreeDepth = %d", nTreeDepth);
     180             :         }
     181             : 
     182          47 :         const int nCellCount = (pabyNodeData[2] << 8) | pabyNodeData[3];
     183          47 :         constexpr int SIZEOF_CELL = 24;  // int64_t + 4 float
     184          47 :         if (nNodeBytes < BLOB_HEADER_SIZE + SIZEOF_CELL * nCellCount)
     185             :         {
     186           0 :             CPLError(CE_Failure, CPLE_AppDefined,
     187             :                      "Not enough bytes when reading node " CPL_FRMT_GIB,
     188             :                      static_cast<GIntBig>(nNodeNo));
     189           0 :             return false;
     190             :         }
     191             : 
     192          47 :         size_t nOffset = BLOB_HEADER_SIZE;
     193          47 :         if (nCurDepth == nTreeDepth)
     194             :         {
     195             :             // Leaf node: it references feature IDs.
     196             : 
     197             :             // If we are about to go above m_nRowGroupSize, flush past
     198             :             // features now, to improve the spatial compacity of the row group.
     199          45 :             if (m_nRowGroupSize > nCellCount &&
     200          45 :                 nCountWrittenFeaturesSinceLastFlush + nCellCount >
     201          45 :                     m_nRowGroupSize)
     202             :             {
     203          14 :                 nCountWrittenFeaturesSinceLastFlush = 0;
     204          14 :                 if (!FlushFeatures())
     205             :                 {
     206           0 :                     return false;
     207             :                 }
     208             :             }
     209             : 
     210             :             // nCellCount shouldn't be over 51 normally, but even 65535
     211             :             // would be fine...
     212          45 :             assert(nCellCount <= 65535);
     213        1247 :             for (int i = 0; i < nCellCount; ++i)
     214             :             {
     215             :                 int64_t nFID;
     216        1202 :                 memcpy(&nFID, pabyNodeData + nOffset, sizeof(int64_t));
     217        1202 :                 CPL_MSBPTR64(&nFID);
     218             : 
     219             :                 const auto poSrcFeature = std::unique_ptr<const OGRFeature>(
     220        1202 :                     m_poTmpGPKGLayer->GetFeature(nFID));
     221        1202 :                 if (!poSrcFeature)
     222             :                 {
     223           0 :                     CPLError(CE_Failure, CPLE_AppDefined,
     224             :                              "Cannot get feature " CPL_FRMT_GIB,
     225             :                              static_cast<GIntBig>(nFID));
     226           0 :                     return false;
     227             :                 }
     228             : 
     229        1202 :                 int nBytesFeature = 0;
     230             :                 const GByte *pabyFeatureData =
     231        1202 :                     poSrcFeature->GetFieldAsBinary(0, &nBytesFeature);
     232        1202 :                 if (!oFeat.DeserializeFromBinary(pabyFeatureData,
     233             :                                                  nBytesFeature))
     234             :                 {
     235           0 :                     CPLError(CE_Failure, CPLE_AppDefined,
     236             :                              "Cannot deserialize feature");
     237           0 :                     return false;
     238             :                 }
     239        1202 :                 if (OGRArrowWriterLayer::ICreateFeature(&oFeat) != OGRERR_NONE)
     240             :                 {
     241           0 :                     return false;
     242             :                 }
     243             : 
     244        1202 :                 nOffset += SIZEOF_CELL;
     245             : 
     246        1202 :                 ++nCountWrittenFeaturesSinceLastFlush;
     247             : 
     248        1202 :                 if ((m_nFeatureCount % PROGRESS_FC_INTERVAL) == 0 ||
     249        1202 :                     m_nFeatureCount == m_nTmpFeatureCount / 2)
     250             :                 {
     251           2 :                     CPLDebugProgress(
     252             :                         "PARQUET",
     253             :                         "CopyTmpGpkgLayerToFinalFile(): %.02f%% progress",
     254           2 :                         100.0 * double(m_nFeatureCount) /
     255           2 :                             double(m_nTmpFeatureCount));
     256             :                 }
     257             :             }
     258             :         }
     259             :         else
     260             :         {
     261             :             // Non-leaf node: it references child nodes.
     262             : 
     263             :             // nCellCount shouldn't be over 51 normally, but even 65535
     264             :             // would be fine...
     265           2 :             assert(nCellCount <= 65535);
     266          46 :             for (int i = 0; i < nCellCount; ++i)
     267             :             {
     268             :                 int64_t nNode;
     269          44 :                 memcpy(&nNode, pabyNodeData + nOffset, sizeof(int64_t));
     270          44 :                 CPL_MSBPTR64(&nNode);
     271             :                 aNodeNoDepthPair.emplace_back(
     272          44 :                     std::make_pair(nNode, nCurDepth + 1));
     273          44 :                 nOffset += SIZEOF_CELL;
     274             :             }
     275             :         }
     276             :     }
     277             : 
     278           3 :     CPLDebug("PARQUET",
     279             :              "CopyTmpGpkgLayerToFinalFile(): 100%%, successfully finished");
     280           3 :     return true;
     281             : }
     282             : 
     283             : /************************************************************************/
     284             : /*                      IsSupportedGeometryType()                       */
     285             : /************************************************************************/
     286             : 
     287         360 : bool OGRParquetWriterLayer::IsSupportedGeometryType(
     288             :     OGRwkbGeometryType eGType) const
     289             : {
     290         360 :     const auto eFlattenType = wkbFlatten(eGType);
     291         360 :     if (!OGR_GT_HasM(eGType) && eFlattenType <= wkbGeometryCollection)
     292             :     {
     293         359 :         return true;
     294             :     }
     295             : 
     296             :     const auto osConfigOptionName =
     297           3 :         "OGR_" + GetDriverUCName() + "_ALLOW_ALL_DIMS";
     298           1 :     if (CPLTestBool(CPLGetConfigOption(osConfigOptionName.c_str(), "NO")))
     299             :     {
     300           0 :         return true;
     301             :     }
     302             : 
     303           1 :     CPLError(CE_Failure, CPLE_NotSupported,
     304             :              "Only 2D and Z geometry types are supported (unless the "
     305             :              "%s configuration option is set to YES)",
     306             :              osConfigOptionName.c_str());
     307           1 :     return false;
     308             : }
     309             : 
     310             : /************************************************************************/
     311             : /*                             SetOptions()                             */
     312             : /************************************************************************/
     313             : 
     314         364 : bool OGRParquetWriterLayer::SetOptions(
     315             :     const OGRGeomFieldDefn *poSrcGeomFieldDefn, CSLConstList papszOptions)
     316             : {
     317         364 :     m_aosCreationOptions = papszOptions;
     318             : 
     319         364 :     const char *pszWriteCoveringBBox = CSLFetchNameValueDef(
     320             :         papszOptions, "WRITE_COVERING_BBOX",
     321             :         CPLGetConfigOption("OGR_PARQUET_WRITE_COVERING_BBOX", nullptr));
     322         364 :     m_bWriteBBoxStruct =
     323         364 :         pszWriteCoveringBBox == nullptr || CPLTestBool(pszWriteCoveringBBox);
     324             : 
     325             :     m_oBBoxStructFieldName =
     326         364 :         CSLFetchNameValueDef(papszOptions, "COVERING_BBOX_NAME", "");
     327             : 
     328         364 :     if (CPLTestBool(CSLFetchNameValueDef(papszOptions, "SORT_BY_BBOX", "NO")))
     329             :     {
     330           8 :         const std::string osTmpGPKG(std::string(m_poDataset->GetDescription()) +
     331           4 :                                     ".tmp.gpkg");
     332           4 :         auto poGPKGDrv = GetGDALDriverManager()->GetDriverByName("GPKG");
     333           4 :         if (!poGPKGDrv)
     334             :         {
     335           1 :             CPLError(
     336             :                 CE_Failure, CPLE_AppDefined,
     337             :                 "Driver GPKG required for SORT_BY_BBOX layer creation option");
     338           1 :             return false;
     339             :         }
     340           3 :         m_poTmpGPKG.reset(poGPKGDrv->Create(osTmpGPKG.c_str(), 0, 0, 0,
     341             :                                             GDT_Unknown, nullptr));
     342           3 :         if (!m_poTmpGPKG)
     343           0 :             return false;
     344           3 :         m_poTmpGPKG->MarkSuppressOnClose();
     345           3 :         m_poTmpGPKGLayer = m_poTmpGPKG->CreateLayer("tmp");
     346          12 :         if (!m_poTmpGPKGLayer ||
     347             :             // Serialized feature
     348           3 :             m_poTmpGPKGLayer->CreateField(
     349           3 :                 std::make_unique<OGRFieldDefn>("serialized_feature", OFTBinary)
     350           3 :                     .get()) != OGRERR_NONE ||
     351             :             // FlushCache is needed to avoid SQLite3 errors on empty layers
     352           9 :             m_poTmpGPKG->FlushCache() != CE_None ||
     353           3 :             m_poTmpGPKGLayer->StartTransaction() != OGRERR_NONE)
     354             :         {
     355           0 :             return false;
     356             :         }
     357             :     }
     358             : 
     359             :     const char *pszGeomEncoding =
     360         363 :         CSLFetchNameValue(papszOptions, "GEOMETRY_ENCODING");
     361         363 :     m_eGeomEncoding = OGRArrowGeomEncoding::WKB;
     362         363 :     if (pszGeomEncoding)
     363             :     {
     364         211 :         if (EQUAL(pszGeomEncoding, "WKB"))
     365           7 :             m_eGeomEncoding = OGRArrowGeomEncoding::WKB;
     366         204 :         else if (EQUAL(pszGeomEncoding, "WKT"))
     367           8 :             m_eGeomEncoding = OGRArrowGeomEncoding::WKT;
     368         196 :         else if (EQUAL(pszGeomEncoding, "GEOARROW_INTERLEAVED"))
     369             :         {
     370          28 :             CPLErrorOnce(
     371             :                 CE_Warning, CPLE_AppDefined,
     372             :                 "Use of GEOMETRY_ENCODING=GEOARROW_INTERLEAVED is not "
     373             :                 "recommended. "
     374             :                 "GeoParquet 1.1 uses GEOMETRY_ENCODING=GEOARROW (struct) "
     375             :                 "instead.");
     376          28 :             m_eGeomEncoding = OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC;
     377             :         }
     378         168 :         else if (EQUAL(pszGeomEncoding, "GEOARROW") ||
     379           0 :                  EQUAL(pszGeomEncoding, "GEOARROW_STRUCT"))
     380         168 :             m_eGeomEncoding = OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC;
     381             :         else
     382             :         {
     383           0 :             CPLError(CE_Failure, CPLE_NotSupported,
     384             :                      "Unsupported GEOMETRY_ENCODING = %s", pszGeomEncoding);
     385           0 :             return false;
     386             :         }
     387             :     }
     388             : 
     389             :     const char *pszCoordPrecision =
     390         363 :         CSLFetchNameValue(papszOptions, "COORDINATE_PRECISION");
     391         363 :     if (pszCoordPrecision)
     392           0 :         m_nWKTCoordinatePrecision = atoi(pszCoordPrecision);
     393             : 
     394         363 :     m_bForceCounterClockwiseOrientation =
     395         363 :         EQUAL(CSLFetchNameValueDef(papszOptions, "POLYGON_ORIENTATION",
     396             :                                    "COUNTERCLOCKWISE"),
     397             :               "COUNTERCLOCKWISE");
     398             : 
     399             :     const auto eGType =
     400         363 :         poSrcGeomFieldDefn ? poSrcGeomFieldDefn->GetType() : wkbNone;
     401         363 :     if (poSrcGeomFieldDefn && eGType != wkbNone)
     402             :     {
     403         334 :         if (!IsSupportedGeometryType(eGType))
     404             :         {
     405           1 :             return false;
     406             :         }
     407             : 
     408         333 :         m_poFeatureDefn->SetGeomType(eGType);
     409         333 :         auto eGeomEncoding = m_eGeomEncoding;
     410         333 :         if (eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC ||
     411         305 :             eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC)
     412             :         {
     413         196 :             const auto eEncodingType = eGeomEncoding;
     414         196 :             eGeomEncoding = GetPreciseArrowGeomEncoding(eEncodingType, eGType);
     415         196 :             if (eGeomEncoding == eEncodingType)
     416           0 :                 return false;
     417             :         }
     418         333 :         m_aeGeomEncoding.push_back(eGeomEncoding);
     419             : 
     420         666 :         std::string osGeometryName;
     421             :         const char *pszGeometryName =
     422         333 :             CSLFetchNameValue(papszOptions, "GEOMETRY_NAME");
     423         333 :         if (pszGeometryName)
     424          16 :             osGeometryName = pszGeometryName;
     425         317 :         else if (poSrcGeomFieldDefn->GetNameRef()[0])
     426           4 :             osGeometryName = poSrcGeomFieldDefn->GetNameRef();
     427             :         else
     428         313 :             osGeometryName = "geometry";
     429         333 :         m_poFeatureDefn->GetGeomFieldDefn(0)->SetName(osGeometryName.c_str());
     430             : 
     431         333 :         const auto poSpatialRef = poSrcGeomFieldDefn->GetSpatialRef();
     432         333 :         if (poSpatialRef)
     433             :         {
     434          42 :             auto poSRS = poSpatialRef->Clone();
     435          42 :             m_poFeatureDefn->GetGeomFieldDefn(0)->SetSpatialRef(poSRS);
     436          42 :             poSRS->Release();
     437             :         }
     438             :     }
     439             : 
     440         362 :     m_osFIDColumn = CSLFetchNameValueDef(papszOptions, "FID", "");
     441             : 
     442         362 :     const char *pszCompression = CSLFetchNameValue(papszOptions, "COMPRESSION");
     443         362 :     if (pszCompression == nullptr)
     444             :     {
     445        1068 :         auto oResult = arrow::util::Codec::GetCompressionType("snappy");
     446         356 :         if (oResult.ok() && arrow::util::Codec::IsAvailable(*oResult))
     447             :         {
     448         356 :             pszCompression = "SNAPPY";
     449             :         }
     450             :         else
     451             :         {
     452           0 :             pszCompression = "NONE";
     453             :         }
     454             :     }
     455             : 
     456         362 :     if (EQUAL(pszCompression, "NONE"))
     457           0 :         pszCompression = "UNCOMPRESSED";
     458             :     auto oResult = arrow::util::Codec::GetCompressionType(
     459         724 :         CPLString(pszCompression).tolower());
     460         362 :     if (!oResult.ok())
     461             :     {
     462           1 :         CPLError(CE_Failure, CPLE_NotSupported,
     463             :                  "Unrecognized compression method: %s", pszCompression);
     464           1 :         return false;
     465             :     }
     466         361 :     m_eCompression = *oResult;
     467         361 :     if (!arrow::util::Codec::IsAvailable(m_eCompression))
     468             :     {
     469           0 :         CPLError(CE_Failure, CPLE_NotSupported,
     470             :                  "Compression method %s is known, but libarrow has not "
     471             :                  "been built with support for it",
     472             :                  pszCompression);
     473           0 :         return false;
     474             :     }
     475         361 :     m_oWriterPropertiesBuilder.compression(m_eCompression);
     476             : 
     477             :     const char *pszCompressionLevel =
     478         361 :         CSLFetchNameValue(papszOptions, "COMPRESSION_LEVEL");
     479         361 :     if (pszCompressionLevel)
     480             :     {
     481           2 :         const int nCompressionLevel = atoi(pszCompressionLevel);
     482           2 :         if (nCompressionLevel != DEFAULT_COMPRESSION_LEVEL)
     483           2 :             m_oWriterPropertiesBuilder.compression_level(nCompressionLevel);
     484             :     }
     485         359 :     else if (EQUAL(pszCompression, "ZSTD"))
     486           1 :         m_oWriterPropertiesBuilder.compression_level(
     487             :             OGR_PARQUET_ZSTD_DEFAULT_COMPRESSION_LEVEL);
     488             : 
     489             :     const std::string osCreator =
     490         361 :         CSLFetchNameValueDef(papszOptions, "CREATOR", "");
     491         361 :     if (!osCreator.empty())
     492           1 :         m_oWriterPropertiesBuilder.created_by(osCreator);
     493             :     else
     494         360 :         m_oWriterPropertiesBuilder.created_by("GDAL " GDAL_RELEASE_NAME
     495             :                                               ", using " CREATED_BY_VERSION);
     496             : 
     497             :     // Undocumented option. Not clear it is useful besides unit test purposes
     498         361 :     if (!CPLTestBool(CSLFetchNameValueDef(papszOptions, "STATISTICS", "YES")))
     499           1 :         m_oWriterPropertiesBuilder.disable_statistics();
     500             : 
     501             : #if PARQUET_VERSION_MAJOR >= 12
     502             :     // Undocumented option. Not clear it is useful to disable it.
     503         361 :     if (CPLTestBool(CSLFetchNameValueDef(papszOptions, "PAGE_INDEX", "YES")))
     504         361 :         m_oWriterPropertiesBuilder.enable_write_page_index();
     505             : #endif
     506             : 
     507             :     const char *pszWriteGeo =
     508         361 :         CPLGetConfigOption("OGR_PARQUET_WRITE_GEO", nullptr);
     509         361 :     m_bWriteGeoMetadata = pszWriteGeo == nullptr || CPLTestBool(pszWriteGeo);
     510             : 
     511         361 :     if (m_eGeomEncoding == OGRArrowGeomEncoding::WKB && eGType != wkbNone)
     512             :     {
     513             : #if ARROW_VERSION_MAJOR >= 21
     514             :         const char *pszUseParquetGeoTypes =
     515             :             CSLFetchNameValueDef(papszOptions, "USE_PARQUET_GEO_TYPES", "NO");
     516             :         if (EQUAL(pszUseParquetGeoTypes, "ONLY"))
     517             :         {
     518             :             m_bUseArrowWKBExtension = true;
     519             :             if (pszWriteGeo == nullptr)
     520             :                 m_bWriteGeoMetadata = false;
     521             :             if (pszWriteCoveringBBox == nullptr)
     522             :                 m_bWriteBBoxStruct = false;
     523             :         }
     524             :         else
     525             :         {
     526             :             m_bUseArrowWKBExtension = CPLTestBool(pszUseParquetGeoTypes);
     527             :         }
     528             : #else
     529         136 :         m_oWriterPropertiesBuilder.disable_statistics(
     530         408 :             parquet::schema::ColumnPath::FromDotString(
     531         136 :                 m_poFeatureDefn->GetGeomFieldDefn(0)->GetNameRef()));
     532             : #endif
     533             :     }
     534             : 
     535             :     const char *pszRowGroupSize =
     536         361 :         CSLFetchNameValue(papszOptions, "ROW_GROUP_SIZE");
     537         361 :     if (pszRowGroupSize)
     538             :     {
     539          19 :         auto nRowGroupSize = static_cast<int64_t>(atoll(pszRowGroupSize));
     540          19 :         if (nRowGroupSize > 0)
     541             :         {
     542          19 :             if (nRowGroupSize > INT_MAX)
     543           0 :                 nRowGroupSize = INT_MAX;
     544          19 :             m_nRowGroupSize = nRowGroupSize;
     545             :         }
     546             :     }
     547             : 
     548         361 :     m_bEdgesSpherical = EQUAL(
     549             :         CSLFetchNameValueDef(papszOptions, "EDGES", "PLANAR"), "SPHERICAL");
     550             : 
     551         361 :     m_bInitializationOK = true;
     552         361 :     return true;
     553             : }
     554             : 
     555             : /************************************************************************/
     556             : /*                          CloseFileWriter()                           */
     557             : /************************************************************************/
     558             : 
     559         361 : bool OGRParquetWriterLayer::CloseFileWriter()
     560             : {
     561         722 :     auto status = m_poFileWriter->Close();
     562         361 :     if (!status.ok())
     563             :     {
     564           0 :         CPLError(CE_Failure, CPLE_AppDefined,
     565             :                  "FileWriter::Close() failed with %s",
     566           0 :                  status.message().c_str());
     567             :     }
     568         722 :     return status.ok();
     569             : }
     570             : 
     571             : /************************************************************************/
     572             : /*                           GetGeoMetadata()                           */
     573             : /************************************************************************/
     574             : 
     575         361 : std::string OGRParquetWriterLayer::GetGeoMetadata() const
     576             : {
     577             :     // Just for unit testing purposes
     578             :     const char *pszGeoMetadata =
     579         361 :         CPLGetConfigOption("OGR_PARQUET_GEO_METADATA", nullptr);
     580         361 :     if (pszGeoMetadata)
     581          16 :         return pszGeoMetadata;
     582             : 
     583         345 :     if (m_poFeatureDefn->GetGeomFieldCount() != 0 && m_bWriteGeoMetadata)
     584             :     {
     585         650 :         CPLJSONObject oRoot;
     586         325 :         oRoot.Add("version", "1.1.0");
     587         325 :         oRoot.Add("primary_column",
     588         325 :                   m_poFeatureDefn->GetGeomFieldDefn(0)->GetNameRef());
     589         650 :         CPLJSONObject oColumns;
     590         325 :         oRoot.Add("columns", oColumns);
     591         667 :         for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
     592             :         {
     593         342 :             const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
     594         684 :             CPLJSONObject oColumn;
     595         342 :             oColumns.Add(poGeomFieldDefn->GetNameRef(), oColumn);
     596         342 :             oColumn.Add("encoding",
     597         342 :                         GetGeomEncodingAsString(m_aeGeomEncoding[i], true));
     598             : 
     599         342 :             if (CPLTestBool(CPLGetConfigOption("OGR_PARQUET_WRITE_CRS", "YES")))
     600             :             {
     601         341 :                 const auto poSRS = poGeomFieldDefn->GetSpatialRef();
     602         341 :                 if (poSRS)
     603             :                 {
     604          82 :                     OGRSpatialReference oSRSIdentified(IdentifyCRS(poSRS));
     605             : 
     606          41 :                     const char *pszAuthName = oSRSIdentified.GetAuthorityName();
     607          41 :                     const char *pszAuthCode = oSRSIdentified.GetAuthorityCode();
     608             : 
     609          41 :                     bool bOmitCRS = false;
     610          41 :                     if (pszAuthName != nullptr && pszAuthCode != nullptr &&
     611          40 :                         ((EQUAL(pszAuthName, "EPSG") &&
     612          37 :                           EQUAL(pszAuthCode, "4326")) ||
     613          22 :                          (EQUAL(pszAuthName, "OGC") &&
     614           3 :                           EQUAL(pszAuthCode, "CRS84"))))
     615             :                     {
     616             :                         // To make things less confusing for non-geo-aware
     617             :                         // consumers, omit EPSG:4326 / OGC:CRS84 CRS by default
     618          21 :                         bOmitCRS = CPLTestBool(CPLGetConfigOption(
     619             :                             "OGR_PARQUET_CRS_OMIT_IF_WGS84", "YES"));
     620             :                     }
     621             : 
     622          41 :                     if (bOmitCRS)
     623             :                     {
     624             :                         // do nothing
     625             :                     }
     626          20 :                     else if (EQUAL(CPLGetConfigOption(
     627             :                                        "OGR_PARQUET_CRS_ENCODING", "PROJJSON"),
     628             :                                    "PROJJSON"))
     629             :                     {
     630             :                         // CRS encoded as PROJJSON for GeoParquet >= 0.4.0
     631          20 :                         char *pszPROJJSON = nullptr;
     632          20 :                         oSRSIdentified.exportToPROJJSON(&pszPROJJSON, nullptr);
     633          40 :                         CPLJSONDocument oCRSDoc;
     634          20 :                         CPL_IGNORE_RET_VAL(oCRSDoc.LoadMemory(pszPROJJSON));
     635          20 :                         CPLFree(pszPROJJSON);
     636          20 :                         CPLJSONObject oCRSRoot = oCRSDoc.GetRoot();
     637          20 :                         RemoveIDFromMemberOfEnsembles(oCRSRoot);
     638          20 :                         oColumn.Add("crs", oCRSRoot);
     639             :                     }
     640             :                     else
     641             :                     {
     642             :                         // WKT was used in GeoParquet <= 0.3.0
     643           0 :                         const char *const apszOptions[] = {
     644             :                             "FORMAT=WKT2_2019", "MULTILINE=NO", nullptr};
     645           0 :                         char *pszWKT = nullptr;
     646           0 :                         oSRSIdentified.exportToWkt(&pszWKT, apszOptions);
     647           0 :                         if (pszWKT)
     648           0 :                             oColumn.Add("crs", pszWKT);
     649           0 :                         CPLFree(pszWKT);
     650             :                     }
     651             : 
     652          41 :                     const double dfCoordEpoch = poSRS->GetCoordinateEpoch();
     653          41 :                     if (dfCoordEpoch > 0)
     654           2 :                         oColumn.Add("epoch", dfCoordEpoch);
     655             :                 }
     656             :                 else
     657             :                 {
     658         300 :                     oColumn.AddNull("crs");
     659             :                 }
     660             :             }
     661             : 
     662         342 :             if (m_bEdgesSpherical)
     663             :             {
     664           3 :                 oColumn.Add("edges", "spherical");
     665             :             }
     666             : 
     667         643 :             if (m_aoEnvelopes[i].IsInit() &&
     668         301 :                 CPLTestBool(
     669             :                     CPLGetConfigOption("OGR_PARQUET_WRITE_BBOX", "YES")))
     670             :             {
     671         301 :                 bool bHasZ = false;
     672         537 :                 for (const auto eGeomType : m_oSetWrittenGeometryTypes[i])
     673             :                 {
     674         343 :                     bHasZ = CPL_TO_BOOL(OGR_GT_HasZ(eGeomType));
     675         343 :                     if (bHasZ)
     676         107 :                         break;
     677             :                 }
     678         301 :                 CPLJSONArray oBBOX;
     679         301 :                 oBBOX.Add(m_aoEnvelopes[i].MinX);
     680         301 :                 oBBOX.Add(m_aoEnvelopes[i].MinY);
     681         301 :                 if (bHasZ)
     682         107 :                     oBBOX.Add(m_aoEnvelopes[i].MinZ);
     683         301 :                 oBBOX.Add(m_aoEnvelopes[i].MaxX);
     684         301 :                 oBBOX.Add(m_aoEnvelopes[i].MaxY);
     685         301 :                 if (bHasZ)
     686         107 :                     oBBOX.Add(m_aoEnvelopes[i].MaxZ);
     687         301 :                 oColumn.Add("bbox", oBBOX);
     688             :             }
     689             : 
     690             :             // Bounding box column definition
     691         614 :             if (m_bWriteBBoxStruct &&
     692         272 :                 CPLTestBool(CPLGetConfigOption(
     693             :                     "OGR_PARQUET_WRITE_COVERING_BBOX_IN_METADATA", "YES")))
     694             :             {
     695         544 :                 CPLJSONObject oCovering;
     696         272 :                 oColumn.Add("covering", oCovering);
     697         544 :                 CPLJSONObject oBBOX;
     698         272 :                 oCovering.Add("bbox", oBBOX);
     699             :                 const auto AddComponent =
     700        3264 :                     [this, i, &oBBOX](const char *pszComponent)
     701             :                 {
     702        1088 :                     CPLJSONArray oArray;
     703        1088 :                     oArray.Add(m_apoFieldsBBOX[i]->name());
     704        1088 :                     oArray.Add(pszComponent);
     705        1088 :                     oBBOX.Add(pszComponent, oArray);
     706        1088 :                 };
     707         272 :                 AddComponent("xmin");
     708         272 :                 AddComponent("ymin");
     709         272 :                 AddComponent("xmax");
     710         272 :                 AddComponent("ymax");
     711             :             }
     712             : 
     713         359 :             const auto GetStringGeometryType = [](OGRwkbGeometryType eType)
     714             :             {
     715         359 :                 const auto eFlattenType = wkbFlatten(eType);
     716         359 :                 std::string osType = "Unknown";
     717         359 :                 if (wkbPoint == eFlattenType)
     718          85 :                     osType = "Point";
     719         274 :                 else if (wkbLineString == eFlattenType)
     720          42 :                     osType = "LineString";
     721         232 :                 else if (wkbPolygon == eFlattenType)
     722          75 :                     osType = "Polygon";
     723         157 :                 else if (wkbMultiPoint == eFlattenType)
     724          34 :                     osType = "MultiPoint";
     725         123 :                 else if (wkbMultiLineString == eFlattenType)
     726          37 :                     osType = "MultiLineString";
     727          86 :                 else if (wkbMultiPolygon == eFlattenType)
     728          81 :                     osType = "MultiPolygon";
     729           5 :                 else if (wkbGeometryCollection == eFlattenType)
     730           5 :                     osType = "GeometryCollection";
     731         359 :                 if (osType != "Unknown")
     732             :                 {
     733             :                     // M and ZM not supported officially currently, but it
     734             :                     // doesn't hurt to anticipate
     735         359 :                     if (OGR_GT_HasZ(eType) && OGR_GT_HasM(eType))
     736           8 :                         osType += " ZM";
     737         351 :                     else if (OGR_GT_HasZ(eType))
     738         115 :                         osType += " Z";
     739         236 :                     else if (OGR_GT_HasM(eType))
     740           8 :                         osType += " M";
     741             :                 }
     742         359 :                 return osType;
     743             :             };
     744             : 
     745         342 :             if (m_bForceCounterClockwiseOrientation)
     746         341 :                 oColumn.Add("orientation", "counterclockwise");
     747             : 
     748         342 :             CPLJSONArray oArray;
     749         701 :             for (const auto eType : m_oSetWrittenGeometryTypes[i])
     750             :             {
     751         359 :                 oArray.Add(GetStringGeometryType(eType));
     752             :             }
     753         342 :             oColumn.Add("geometry_types", oArray);
     754             :         }
     755             : 
     756         325 :         return oRoot.Format(CPLJSONObject::PrettyFormat::Plain);
     757             :     }
     758          20 :     return std::string();
     759             : }
     760             : 
     761             : /************************************************************************/
     762             : /*                 PerformStepsBeforeFinalFlushGroup()                  */
     763             : /************************************************************************/
     764             : 
     765         361 : void OGRParquetWriterLayer::PerformStepsBeforeFinalFlushGroup()
     766             : {
     767         361 :     if (m_poKeyValueMetadata)
     768             :     {
     769         722 :         std::string osGeoMetadata = GetGeoMetadata();
     770         722 :         auto poTmpSchema = m_poSchema;
     771         361 :         if (!osGeoMetadata.empty())
     772             :         {
     773             :             // HACK: it would be good for Arrow to provide a clean way to alter
     774             :             // key value metadata before finalizing.
     775             :             // We need to write metadata at end to write the bounding box.
     776         341 :             const_cast<arrow::KeyValueMetadata *>(m_poKeyValueMetadata.get())
     777         341 :                 ->Append("geo", osGeoMetadata);
     778             : 
     779         341 :             auto kvMetadata = poTmpSchema->metadata()
     780          15 :                                   ? poTmpSchema->metadata()->Copy()
     781         356 :                                   : std::make_shared<arrow::KeyValueMetadata>();
     782         341 :             kvMetadata->Append("geo", std::move(osGeoMetadata));
     783         341 :             poTmpSchema = poTmpSchema->WithMetadata(kvMetadata);
     784             :         }
     785             : 
     786         361 :         if (CPLTestBool(
     787             :                 CPLGetConfigOption("OGR_PARQUET_WRITE_ARROW_SCHEMA", "YES")))
     788             :         {
     789             :             auto status =
     790         722 :                 ::arrow::ipc::SerializeSchema(*poTmpSchema, m_poMemoryPool);
     791         361 :             if (status.ok())
     792             :             {
     793             :                 // The serialized schema is not UTF-8, which is required for
     794             :                 // Thrift
     795         722 :                 const std::string schema_as_string = (*status)->ToString();
     796             :                 std::string schema_base64 =
     797         361 :                     ::arrow::util::base64_encode(schema_as_string);
     798         361 :                 static const std::string kArrowSchemaKey = "ARROW:schema";
     799             :                 const_cast<arrow::KeyValueMetadata *>(
     800         361 :                     m_poKeyValueMetadata.get())
     801         361 :                     ->Append(kArrowSchemaKey, std::move(schema_base64));
     802             :             }
     803             :         }
     804             : 
     805             :         // Put GDAL metadata into a gdal:metadata domain
     806         722 :         CPLJSONObject oMultiMetadata;
     807         361 :         bool bHasMultiMetadata = false;
     808         369 :         auto &l_oMDMD = oMDMD.GetDomainList() && *(oMDMD.GetDomainList())
     809         369 :                             ? oMDMD
     810         353 :                             : m_poDataset->GetMultiDomainMetadata();
     811         371 :         for (CSLConstList papszDomainIter = l_oMDMD.GetDomainList();
     812         371 :              papszDomainIter && *papszDomainIter; ++papszDomainIter)
     813             :         {
     814          10 :             const char *pszDomain = *papszDomainIter;
     815          10 :             CSLConstList papszMD = l_oMDMD.GetMetadata(pszDomain);
     816          10 :             if (STARTS_WITH(pszDomain, "json:") && papszMD && papszMD[0])
     817             :             {
     818           1 :                 CPLJSONDocument oDoc;
     819           1 :                 if (oDoc.LoadMemory(papszMD[0]))
     820             :                 {
     821           1 :                     bHasMultiMetadata = true;
     822           1 :                     oMultiMetadata.Add(pszDomain, oDoc.GetRoot());
     823           1 :                     continue;
     824           0 :                 }
     825             :             }
     826           9 :             else if (STARTS_WITH(pszDomain, "xml:") && papszMD && papszMD[0])
     827             :             {
     828           1 :                 bHasMultiMetadata = true;
     829           1 :                 oMultiMetadata.Add(pszDomain, papszMD[0]);
     830           1 :                 continue;
     831             :             }
     832          16 :             CPLJSONObject oMetadata;
     833           8 :             bool bHasMetadata = false;
     834          16 :             for (CSLConstList papszMDIter = papszMD;
     835          16 :                  papszMDIter && *papszMDIter; ++papszMDIter)
     836             :             {
     837           8 :                 char *pszKey = nullptr;
     838           8 :                 const char *pszValue = CPLParseNameValue(*papszMDIter, &pszKey);
     839           8 :                 if (pszKey && pszValue)
     840             :                 {
     841           8 :                     bHasMetadata = true;
     842           8 :                     bHasMultiMetadata = true;
     843           8 :                     oMetadata.Add(pszKey, pszValue);
     844             :                 }
     845           8 :                 CPLFree(pszKey);
     846             :             }
     847           8 :             if (bHasMetadata)
     848           8 :                 oMultiMetadata.Add(pszDomain, oMetadata);
     849             :         }
     850         361 :         if (bHasMultiMetadata)
     851             :         {
     852           8 :             const_cast<arrow::KeyValueMetadata *>(m_poKeyValueMetadata.get())
     853           8 :                 ->Append(
     854             :                     "gdal:metadata",
     855          16 :                     oMultiMetadata.Format(CPLJSONObject::PrettyFormat::Plain));
     856             :         }
     857             : 
     858         361 :         if (!m_aosCreationOptions.empty())
     859             :         {
     860         508 :             CPLJSONObject oCreationOptions;
     861         254 :             bool bEmpty = true;
     862        1120 :             for (const auto &[key, value] :
     863        1374 :                  cpl::IterateNameValue(m_aosCreationOptions))
     864             :             {
     865         560 :                 if (!EQUAL(key, "FID") && !EQUAL(key, "GEOMETRY_NAME") &&
     866         521 :                     !EQUAL(key, "EDGES"))
     867             :                 {
     868         517 :                     bEmpty = false;
     869         517 :                     oCreationOptions.Add(key, value);
     870             :                 }
     871             :             }
     872         254 :             if (!bEmpty)
     873             :             {
     874             :                 const_cast<arrow::KeyValueMetadata *>(
     875         240 :                     m_poKeyValueMetadata.get())
     876         240 :                     ->Append("gdal:creation-options",
     877         480 :                              oCreationOptions.Format(
     878             :                                  CPLJSONObject::PrettyFormat::Plain));
     879             :             }
     880             :         }
     881             :     }
     882         361 : }
     883             : 
     884             : /************************************************************************/
     885             : /*                                Open()                                */
     886             : /************************************************************************/
     887             : 
     888             : // Same as parquet::arrow::FileWriter::Open(), except we also
     889             : // return KeyValueMetadata
     890             : static arrow::Status
     891         361 : Open(const ::arrow::Schema &schema, ::arrow::MemoryPool *pool,
     892             :      std::shared_ptr<::arrow::io::OutputStream> sink,
     893             :      std::shared_ptr<parquet::WriterProperties> properties,
     894             :      std::shared_ptr<parquet::ArrowWriterProperties> arrow_properties,
     895             :      std::unique_ptr<parquet::arrow::FileWriter> *writer,
     896             :      std::shared_ptr<const arrow::KeyValueMetadata> *outMetadata)
     897             : {
     898         361 :     std::shared_ptr<parquet::SchemaDescriptor> parquet_schema;
     899         722 :     RETURN_NOT_OK(parquet::arrow::ToParquetSchema(
     900             :         &schema, *properties, *arrow_properties, &parquet_schema));
     901             : 
     902             :     auto schema_node = std::static_pointer_cast<parquet::schema::GroupNode>(
     903         722 :         parquet_schema->schema_root());
     904             : 
     905         361 :     auto metadata = schema.metadata()
     906          20 :                         ? schema.metadata()->Copy()
     907         742 :                         : std::make_shared<arrow::KeyValueMetadata>();
     908         361 :     *outMetadata = metadata;
     909             : 
     910         361 :     std::unique_ptr<parquet::ParquetFileWriter> base_writer;
     911         361 :     PARQUET_CATCH_NOT_OK(base_writer = parquet::ParquetFileWriter::Open(
     912             :                              std::move(sink), std::move(schema_node),
     913             :                              std::move(properties), metadata));
     914             : 
     915         361 :     auto schema_ptr = std::make_shared<::arrow::Schema>(schema);
     916             :     return parquet::arrow::FileWriter::Make(
     917         722 :         pool, std::move(base_writer), std::move(schema_ptr),
     918        1083 :         std::move(arrow_properties), writer);
     919             : }
     920             : 
     921             : /************************************************************************/
     922             : /*                            CreateSchema()                            */
     923             : /************************************************************************/
     924             : 
     925         361 : void OGRParquetWriterLayer::CreateSchema()
     926             : {
     927         361 :     CreateSchemaCommon();
     928         361 : }
     929             : 
     930             : /************************************************************************/
     931             : /*                          CreateGeomField()                           */
     932             : /************************************************************************/
     933             : 
     934          27 : OGRErr OGRParquetWriterLayer::CreateGeomField(const OGRGeomFieldDefn *poField,
     935             :                                               int bApproxOK)
     936             : {
     937          27 :     OGRErr eErr = OGRArrowWriterLayer::CreateGeomField(poField, bApproxOK);
     938          53 :     if (eErr == OGRERR_NONE &&
     939          26 :         m_aeGeomEncoding.back() == OGRArrowGeomEncoding::WKB
     940             : #if ARROW_VERSION_MAJOR < 21
     941             :         // Geostatistics in Arrow 21 do not support geographic type for now
     942          53 :         && m_bEdgesSpherical
     943             : #endif
     944             :     )
     945             :     {
     946           0 :         m_oWriterPropertiesBuilder.disable_statistics(
     947           0 :             parquet::schema::ColumnPath::FromDotString(
     948           0 :                 m_poFeatureDefn
     949           0 :                     ->GetGeomFieldDefn(m_poFeatureDefn->GetGeomFieldCount() - 1)
     950             :                     ->GetNameRef()));
     951             :     }
     952          27 :     return eErr;
     953             : }
     954             : 
     955             : /************************************************************************/
     956             : /*                            CreateWriter()                            */
     957             : /************************************************************************/
     958             : 
     959         361 : void OGRParquetWriterLayer::CreateWriter()
     960             : {
     961         361 :     CPLAssert(m_poFileWriter == nullptr);
     962             : 
     963         361 :     if (m_poSchema == nullptr)
     964             :     {
     965          43 :         CreateSchema();
     966             :     }
     967             :     else
     968             :     {
     969         318 :         FinalizeSchema();
     970             :     }
     971             : 
     972             :     auto arrowWriterProperties =
     973         361 :         parquet::ArrowWriterProperties::Builder().store_schema()->build();
     974        1083 :     CPL_IGNORE_RET_VAL(Open(*m_poSchema, m_poMemoryPool, m_poOutputStream,
     975         722 :                             m_oWriterPropertiesBuilder.build(),
     976         361 :                             std::move(arrowWriterProperties), &m_poFileWriter,
     977             :                             &m_poKeyValueMetadata));
     978         361 : }
     979             : 
     980             : /************************************************************************/
     981             : /*                           ICreateFeature()                           */
     982             : /************************************************************************/
     983             : 
     984        3390 : OGRErr OGRParquetWriterLayer::ICreateFeature(OGRFeature *poFeature)
     985             : {
     986             :     // If not using SORT_BY_BBOX=YES layer creation option, we can directly
     987             :     // write features to the final Parquet file
     988        3390 :     if (!m_poTmpGPKGLayer)
     989        1186 :         return OGRArrowWriterLayer::ICreateFeature(poFeature);
     990             : 
     991             :     // SORT_BY_BBOX=YES case: we write for now a serialized version of poFeature
     992             :     // in a temporary GeoPackage file.
     993             : 
     994        2204 :     GIntBig nFID = poFeature->GetFID();
     995        2204 :     if (!m_osFIDColumn.empty() && nFID == OGRNullFID)
     996             :     {
     997        1102 :         nFID = m_nTmpFeatureCount;
     998        1102 :         poFeature->SetFID(nFID);
     999             :     }
    1000        2204 :     ++m_nTmpFeatureCount;
    1001             : 
    1002        4408 :     std::vector<GByte> abyBuffer;
    1003             :     // Serialize the source feature as a single array of bytes to preserve it
    1004             :     // fully
    1005        2204 :     if (!poFeature->SerializeToBinary(abyBuffer))
    1006             :     {
    1007           0 :         return OGRERR_FAILURE;
    1008             :     }
    1009             : 
    1010             :     // SQLite3 limitation: a row must fit in slightly less than 1 GB.
    1011        2204 :     constexpr int SOME_MARGIN = 128;
    1012        2204 :     if (abyBuffer.size() > 1024 * 1024 * 1024 - SOME_MARGIN)
    1013             :     {
    1014           0 :         CPLError(CE_Failure, CPLE_NotSupported,
    1015             :                  "Features larger than 1 GB are not supported");
    1016           0 :         return OGRERR_FAILURE;
    1017             :     }
    1018             : 
    1019        4408 :     OGRFeature oFeat(m_poTmpGPKGLayer->GetLayerDefn());
    1020        2204 :     oFeat.SetFID(nFID);
    1021        2204 :     oFeat.SetField(0, static_cast<int>(abyBuffer.size()), abyBuffer.data());
    1022        2204 :     const auto poSrcGeom = poFeature->GetGeometryRef();
    1023        2204 :     if (poSrcGeom && !poSrcGeom->IsEmpty())
    1024             :     {
    1025             :         // For the purpose of building an RTree, just use the bounding box of
    1026             :         // the geometry as the geometry.
    1027        1202 :         OGREnvelope sEnvelope;
    1028        1202 :         poSrcGeom->getEnvelope(&sEnvelope);
    1029        2404 :         auto poPoly = std::make_unique<OGRPolygon>();
    1030        2404 :         auto poLR = std::make_unique<OGRLinearRing>();
    1031        1202 :         poLR->addPoint(sEnvelope.MinX, sEnvelope.MinY);
    1032        1202 :         poLR->addPoint(sEnvelope.MinX, sEnvelope.MaxY);
    1033        1202 :         poLR->addPoint(sEnvelope.MaxX, sEnvelope.MaxY);
    1034        1202 :         poLR->addPoint(sEnvelope.MaxX, sEnvelope.MinY);
    1035        1202 :         poLR->addPoint(sEnvelope.MinX, sEnvelope.MinY);
    1036        1202 :         poPoly->addRingDirectly(poLR.release());
    1037        1202 :         oFeat.SetGeometryDirectly(poPoly.release());
    1038             :     }
    1039        2204 :     return m_poTmpGPKGLayer->CreateFeature(&oFeat);
    1040             : }
    1041             : 
    1042             : /************************************************************************/
    1043             : /*                             FlushGroup()                             */
    1044             : /************************************************************************/
    1045             : 
    1046         333 : bool OGRParquetWriterLayer::FlushGroup()
    1047             : {
    1048             : #if PARQUET_VERSION_MAJOR >= 20
    1049             :     auto status = m_poFileWriter->NewRowGroup();
    1050             : #else
    1051         666 :     auto status = m_poFileWriter->NewRowGroup(m_apoBuilders[0]->length());
    1052             : #endif
    1053         333 :     if (!status.ok())
    1054             :     {
    1055           0 :         CPLError(CE_Failure, CPLE_AppDefined, "NewRowGroup() failed with %s",
    1056           0 :                  status.message().c_str());
    1057           0 :         ClearArrayBuilers();
    1058           0 :         return false;
    1059             :     }
    1060             : 
    1061         333 :     auto ret = WriteArrays(
    1062        1291 :         [this](const std::shared_ptr<arrow::Field> &field,
    1063        1291 :                const std::shared_ptr<arrow::Array> &array)
    1064             :         {
    1065        2582 :             auto l_status = m_poFileWriter->WriteColumnChunk(*array);
    1066        1291 :             if (!l_status.ok())
    1067             :             {
    1068           0 :                 CPLError(CE_Failure, CPLE_AppDefined,
    1069             :                          "WriteColumnChunk() failed for field %s: %s",
    1070           0 :                          field->name().c_str(), l_status.message().c_str());
    1071           0 :                 return false;
    1072             :             }
    1073        1291 :             return true;
    1074             :         });
    1075             : 
    1076         333 :     ClearArrayBuilers();
    1077         333 :     return ret;
    1078             : }
    1079             : 
    1080             : /************************************************************************/
    1081             : /*                   FixupWKBGeometryBeforeWriting()                    */
    1082             : /************************************************************************/
    1083             : 
    1084          51 : void OGRParquetWriterLayer::FixupWKBGeometryBeforeWriting(GByte *pabyWkb,
    1085             :                                                           size_t nLen)
    1086             : {
    1087          51 :     if (!m_bForceCounterClockwiseOrientation)
    1088           0 :         return;
    1089             : 
    1090          51 :     OGRWKBFixupCounterClockWiseExternalRing(pabyWkb, nLen);
    1091             : }
    1092             : 
    1093             : /************************************************************************/
    1094             : /*                     FixupGeometryBeforeWriting()                     */
    1095             : /************************************************************************/
    1096             : 
    1097        1379 : void OGRParquetWriterLayer::FixupGeometryBeforeWriting(OGRGeometry *poGeom)
    1098             : {
    1099        1379 :     if (!m_bForceCounterClockwiseOrientation)
    1100           3 :         return;
    1101             : 
    1102        1376 :     const auto eFlattenType = wkbFlatten(poGeom->getGeometryType());
    1103             :     // Polygon rings MUST follow the right-hand rule for orientation
    1104             :     // (counterclockwise external rings, clockwise internal rings)
    1105        1376 :     if (eFlattenType == wkbPolygon)
    1106             :     {
    1107          74 :         bool bFirstRing = true;
    1108         151 :         for (auto poRing : poGeom->toPolygon())
    1109             :         {
    1110          85 :             if ((bFirstRing && poRing->isClockwise()) ||
    1111           8 :                 (!bFirstRing && !poRing->isClockwise()))
    1112             :             {
    1113          72 :                 poRing->reversePoints();
    1114             :             }
    1115          77 :             bFirstRing = false;
    1116             :         }
    1117             :     }
    1118        1302 :     else if (eFlattenType == wkbMultiPolygon ||
    1119             :              eFlattenType == wkbGeometryCollection)
    1120             :     {
    1121          35 :         for (auto poSubGeom : poGeom->toGeometryCollection())
    1122             :         {
    1123          21 :             FixupGeometryBeforeWriting(poSubGeom);
    1124             :         }
    1125             :     }
    1126             : }
    1127             : 
    1128             : /************************************************************************/
    1129             : /*                          WriteArrowBatch()                           */
    1130             : /************************************************************************/
    1131             : 
    1132             : #if PARQUET_VERSION_MAJOR > 10
    1133             : inline bool
    1134          25 : OGRParquetWriterLayer::WriteArrowBatch(const struct ArrowSchema *schema,
    1135             :                                        struct ArrowArray *array,
    1136             :                                        CSLConstList papszOptions)
    1137             : {
    1138          25 :     if (m_poTmpGPKGLayer)
    1139             :     {
    1140             :         // When using SORT_BY_BBOX=YES option, we can't directly write the
    1141             :         // input array, because we need to sort features. Hence we fallback
    1142             :         // to the OGRLayer base implementation, which will ultimately call
    1143             :         // OGRParquetWriterLayer::ICreateFeature()
    1144           0 :         return OGRLayer::WriteArrowBatch(schema, array, papszOptions);
    1145             :     }
    1146             : 
    1147          50 :     return WriteArrowBatchInternal(
    1148             :         schema, array, papszOptions,
    1149          50 :         [this](const std::shared_ptr<arrow::RecordBatch> &poBatch)
    1150             :         {
    1151          50 :             auto status = m_poFileWriter->NewBufferedRowGroup();
    1152          25 :             if (!status.ok())
    1153             :             {
    1154           0 :                 CPLError(CE_Failure, CPLE_AppDefined,
    1155             :                          "NewBufferedRowGroup() failed with %s",
    1156           0 :                          status.message().c_str());
    1157           0 :                 return false;
    1158             :             }
    1159             : 
    1160          25 :             status = m_poFileWriter->WriteRecordBatch(*poBatch);
    1161          25 :             if (!status.ok())
    1162             :             {
    1163           0 :                 CPLError(CE_Failure, CPLE_AppDefined,
    1164             :                          "WriteRecordBatch() failed: %s",
    1165           0 :                          status.message().c_str());
    1166           0 :                 return false;
    1167             :             }
    1168             : 
    1169          25 :             return true;
    1170          25 :         });
    1171             : }
    1172             : #endif
    1173             : 
    1174             : /************************************************************************/
    1175             : /*                           TestCapability()                           */
    1176             : /************************************************************************/
    1177             : 
    1178         694 : inline int OGRParquetWriterLayer::TestCapability(const char *pszCap) const
    1179             : {
    1180             : #if PARQUET_VERSION_MAJOR <= 10
    1181             :     if (EQUAL(pszCap, OLCFastWriteArrowBatch))
    1182             :         return false;
    1183             : #endif
    1184             : 
    1185         694 :     if (m_poTmpGPKGLayer && EQUAL(pszCap, OLCFastWriteArrowBatch))
    1186             :     {
    1187             :         // When using SORT_BY_BBOX=YES option, we can't directly write the
    1188             :         // input array, because we need to sort features. So this is not
    1189             :         // fast
    1190           1 :         return false;
    1191             :     }
    1192             : 
    1193         693 :     return OGRArrowWriterLayer::TestCapability(pszCap);
    1194             : }
    1195             : 
    1196             : /************************************************************************/
    1197             : /*                     CreateFieldFromArrowSchema()                     */
    1198             : /************************************************************************/
    1199             : 
    1200             : #if PARQUET_VERSION_MAJOR > 10
    1201         479 : bool OGRParquetWriterLayer::CreateFieldFromArrowSchema(
    1202             :     const struct ArrowSchema *schema, CSLConstList papszOptions)
    1203             : {
    1204         479 :     if (m_poTmpGPKGLayer)
    1205             :     {
    1206             :         // When using SORT_BY_BBOX=YES option, we can't directly write the
    1207             :         // input array, because we need to sort features. But this process
    1208             :         // only supports the base Arrow types supported by
    1209             :         // OGRLayer::WriteArrowBatch()
    1210           0 :         return OGRLayer::CreateFieldFromArrowSchema(schema, papszOptions);
    1211             :     }
    1212             : 
    1213         479 :     return OGRArrowWriterLayer::CreateFieldFromArrowSchema(schema,
    1214         479 :                                                            papszOptions);
    1215             : }
    1216             : #endif
    1217             : 
    1218             : /************************************************************************/
    1219             : /*                       IsArrowSchemaSupported()                       */
    1220             : /************************************************************************/
    1221             : 
    1222             : #if PARQUET_VERSION_MAJOR > 10
    1223        1095 : bool OGRParquetWriterLayer::IsArrowSchemaSupported(
    1224             :     const struct ArrowSchema *schema, CSLConstList papszOptions,
    1225             :     std::string &osErrorMsg) const
    1226             : {
    1227        1095 :     if (m_poTmpGPKGLayer)
    1228             :     {
    1229             :         // When using SORT_BY_BBOX=YES option, we can't directly write the
    1230             :         // input array, because we need to sort features. But this process
    1231             :         // only supports the base Arrow types supported by
    1232             :         // OGRLayer::WriteArrowBatch()
    1233           0 :         return OGRLayer::IsArrowSchemaSupported(schema, papszOptions,
    1234           0 :                                                 osErrorMsg);
    1235             :     }
    1236             : 
    1237        1095 :     if (schema->format[0] == 'e' && schema->format[1] == 0)
    1238             :     {
    1239           1 :         osErrorMsg = "float16 not supported";
    1240           1 :         return false;
    1241             :     }
    1242        1094 :     if (schema->format[0] == 'v' && schema->format[1] == 'u')
    1243             :     {
    1244           1 :         osErrorMsg = "StringView not supported";
    1245           1 :         return false;
    1246             :     }
    1247        1093 :     if (schema->format[0] == 'v' && schema->format[1] == 'z')
    1248             :     {
    1249           1 :         osErrorMsg = "BinaryView not supported";
    1250           1 :         return false;
    1251             :     }
    1252        1092 :     if (schema->format[0] == '+' && schema->format[1] == 'v')
    1253             :     {
    1254           0 :         if (schema->format[2] == 'l')
    1255             :         {
    1256           0 :             osErrorMsg = "ListView not supported";
    1257           0 :             return false;
    1258             :         }
    1259           0 :         else if (schema->format[2] == 'L')
    1260             :         {
    1261           0 :             osErrorMsg = "LargeListView not supported";
    1262           0 :             return false;
    1263             :         }
    1264             :     }
    1265        2169 :     for (int64_t i = 0; i < schema->n_children; ++i)
    1266             :     {
    1267        1080 :         if (!IsArrowSchemaSupported(schema->children[i], papszOptions,
    1268             :                                     osErrorMsg))
    1269             :         {
    1270           3 :             return false;
    1271             :         }
    1272             :     }
    1273        1089 :     return true;
    1274             : }
    1275             : #endif
    1276             : 
    1277             : /************************************************************************/
    1278             : /*                            SetMetadata()                             */
    1279             : /************************************************************************/
    1280             : 
    1281          13 : CPLErr OGRParquetWriterLayer::SetMetadata(CSLConstList papszMetadata,
    1282             :                                           const char *pszDomain)
    1283             : {
    1284          13 :     if (!pszDomain || !EQUAL(pszDomain, "SHAPEFILE"))
    1285             :     {
    1286           8 :         return OGRLayer::SetMetadata(papszMetadata, pszDomain);
    1287             :     }
    1288           5 :     return CE_None;
    1289             : }
    1290             : 
    1291             : /************************************************************************/
    1292             : /*                             GetDataset()                             */
    1293             : /************************************************************************/
    1294             : 
    1295          26 : GDALDataset *OGRParquetWriterLayer::GetDataset()
    1296             : {
    1297          26 :     return m_poDataset;
    1298             : }