Line data    Source code

       1             : /******************************************************************************
       2             :  *
       3             :  * Project:  Parquet Translator
       4             :  * Purpose:  Implements OGRParquetDriver.
       5             :  * Author:   Even Rouault, <even.rouault at spatialys.com>
       6             :  *
       7             :  ******************************************************************************
       8             :  * Copyright (c) 2022, Planet Labs
       9             :  *
      10             :  * SPDX-License-Identifier: MIT
      11             :  ****************************************************************************/
      12             : 
      13             : #ifdef STANDALONE
      14             : #include "gdal_version.h"
      15             : #else
      16             : #undef DO_NOT_DEFINE_GDAL_DATE_NAME
      17             : #include "gdal_version_full/gdal_version.h"
      18             : #endif
      19             : 
      20             : #include "ogr_parquet.h"
      21             : 
      22             : #include "../arrow_common/ograrrowwriterlayer.hpp"
      23             : 
      24             : #include "ogr_wkb.h"
      25             : 
      26             : #include <cassert>
      27             : #include <utility>
      28             : 
      29             : /************************************************************************/
      30             : /*                      OGRParquetWriterLayer()                         */
      31             : /************************************************************************/
      32             : 
      33         272 : OGRParquetWriterLayer::OGRParquetWriterLayer(
      34             :     OGRParquetWriterDataset *poDataset, arrow::MemoryPool *poMemoryPool,
      35             :     const std::shared_ptr<arrow::io::OutputStream> &poOutputStream,
      36         272 :     const char *pszLayerName)
      37             :     : OGRArrowWriterLayer(poMemoryPool, poOutputStream, pszLayerName),
      38         272 :       m_poDataset(poDataset)
      39             : {
      40         272 :     m_bWriteFieldArrowExtensionName = CPLTestBool(
      41             :         CPLGetConfigOption("OGR_PARQUET_WRITE_ARROW_EXTENSION_NAME", "NO"));
      42         272 : }
      43             : 
      44             : /************************************************************************/
      45             : /*                                Close()                               */
      46             : /************************************************************************/
      47             : 
      48         269 : bool OGRParquetWriterLayer::Close()
      49             : {
      50         269 :     if (m_poTmpGPKGLayer)
      51             :     {
      52           2 :         if (!CopyTmpGpkgLayerToFinalFile())
      53           0 :             return false;
      54             :     }
      55             : 
      56         269 :     if (m_bInitializationOK)
      57             :     {
      58         269 :         if (!FinalizeWriting())
      59           0 :             return false;
      60             :     }
      61             : 
      62         269 :     return true;
      63             : }
      64             : 
      65             : /************************************************************************/
      66             : /*                     CopyTmpGpkgLayerToFinalFile()                    */
      67             : /************************************************************************/
      68             : 
      69           2 : bool OGRParquetWriterLayer::CopyTmpGpkgLayerToFinalFile()
      70             : {
      71           2 :     if (!m_poTmpGPKGLayer)
      72             :     {
      73           0 :         return true;
      74             :     }
      75             : 
      76           2 :     CPLDebug("PARQUET", "CopyTmpGpkgLayerToFinalFile(): start...");
      77             : 
      78           2 :     VSIUnlink(m_poTmpGPKG->GetDescription());
      79             : 
      80           4 :     OGRFeature oFeat(m_poFeatureDefn);
      81             : 
      82             :     // Interval in terms of features between 2 debug progress report messages
      83           2 :     constexpr int PROGRESS_FC_INTERVAL = 100 * 1000;
      84             : 
      85             :     // First, write features without geometries
      86             :     {
      87           2 :         auto poTmpLayer = std::unique_ptr<OGRLayer>(m_poTmpGPKG->ExecuteSQL(
      88             :             "SELECT serialized_feature FROM tmp WHERE fid NOT IN (SELECT id "
      89             :             "FROM rtree_tmp_geom)",
      90           2 :             nullptr, nullptr));
      91           2 :         if (!poTmpLayer)
      92           0 :             return false;
      93        1004 :         for (const auto &poSrcFeature : poTmpLayer.get())
      94             :         {
      95        1002 :             int nBytesFeature = 0;
      96             :             const GByte *pabyFeatureData =
      97        1002 :                 poSrcFeature->GetFieldAsBinary(0, &nBytesFeature);
      98        1002 :             if (!oFeat.DeserializeFromBinary(pabyFeatureData, nBytesFeature))
      99             :             {
     100           0 :                 CPLError(CE_Failure, CPLE_AppDefined,
     101             :                          "Cannot deserialize feature");
     102           0 :                 return false;
     103             :             }
     104        1002 :             if (OGRArrowWriterLayer::ICreateFeature(&oFeat) != OGRERR_NONE)
     105             :             {
     106           0 :                 return false;
     107             :             }
     108             : 
     109        1002 :             if ((m_nFeatureCount % PROGRESS_FC_INTERVAL) == 0)
     110             :             {
     111           0 :                 CPLDebugProgress(
     112             :                     "PARQUET",
     113             :                     "CopyTmpGpkgLayerToFinalFile(): %.02f%% progress",
     114           0 :                     100.0 * double(m_nFeatureCount) /
     115           0 :                         double(m_nTmpFeatureCount));
     116             :             }
     117             :         }
     118             : 
     119           2 :         if (!FlushFeatures())
     120             :         {
     121           0 :             return false;
     122             :         }
     123             :     }
     124             : 
     125             :     // Now walk through the GPKG RTree for features with geometries
     126             :     // Cf https://github.com/sqlite/sqlite/blob/master/ext/rtree/rtree.c
     127             :     // for the description of the content of the rtree _node table
     128           4 :     std::vector<std::pair<int64_t, int>> aNodeNoDepthPair;
     129           2 :     int nTreeDepth = 0;
     130             :     // Queue the root node
     131             :     aNodeNoDepthPair.emplace_back(
     132           2 :         std::make_pair(/* nodeNo = */ 1, /* depth = */ 0));
     133           2 :     int nCountWrittenFeaturesSinceLastFlush = 0;
     134          50 :     while (!aNodeNoDepthPair.empty())
     135             :     {
     136          48 :         const auto &oLastPair = aNodeNoDepthPair.back();
     137          48 :         const int64_t nNodeNo = oLastPair.first;
     138          48 :         const int nCurDepth = oLastPair.second;
     139             :         //CPLDebug("PARQUET", "Reading nodeNode=%d, curDepth=%d", int(nNodeNo), nCurDepth);
     140          48 :         aNodeNoDepthPair.pop_back();
     141             : 
     142          48 :         auto poRTreeLayer = std::unique_ptr<OGRLayer>(m_poTmpGPKG->ExecuteSQL(
     143             :             CPLSPrintf("SELECT data FROM rtree_tmp_geom_node WHERE nodeno "
     144             :                        "= " CPL_FRMT_GIB,
     145             :                        static_cast<GIntBig>(nNodeNo)),
     146          48 :             nullptr, nullptr));
     147          48 :         if (!poRTreeLayer)
     148             :         {
     149           0 :             CPLError(CE_Failure, CPLE_AppDefined,
     150             :                      "Cannot read node " CPL_FRMT_GIB,
     151             :                      static_cast<GIntBig>(nNodeNo));
     152           0 :             return false;
     153             :         }
     154             :         const auto poRTreeFeature =
     155          48 :             std::unique_ptr<const OGRFeature>(poRTreeLayer->GetNextFeature());
     156          48 :         if (!poRTreeFeature)
     157             :         {
     158           0 :             CPLError(CE_Failure, CPLE_AppDefined,
     159             :                      "Cannot read node " CPL_FRMT_GIB,
     160             :                      static_cast<GIntBig>(nNodeNo));
     161           0 :             return false;
     162             :         }
     163             : 
     164          48 :         int nNodeBytes = 0;
     165             :         const GByte *pabyNodeData =
     166          48 :             poRTreeFeature->GetFieldAsBinary(0, &nNodeBytes);
     167          48 :         constexpr int BLOB_HEADER_SIZE = 4;
     168          48 :         if (nNodeBytes < BLOB_HEADER_SIZE)
     169             :         {
     170           0 :             CPLError(CE_Failure, CPLE_AppDefined,
     171             :                      "Not enough bytes when reading node " CPL_FRMT_GIB,
     172             :                      static_cast<GIntBig>(nNodeNo));
     173           0 :             return false;
     174             :         }
     175          48 :         if (nNodeNo == 1)
     176             :         {
     177             :             // Get the RTree depth from the root node
     178           2 :             nTreeDepth = (pabyNodeData[0] << 8) | pabyNodeData[1];
     179             :             //CPLDebug("PARQUET", "nTreeDepth = %d", nTreeDepth);
     180             :         }
     181             : 
     182          48 :         const int nCellCount = (pabyNodeData[2] << 8) | pabyNodeData[3];
     183          48 :         constexpr int SIZEOF_CELL = 24;  // int64_t + 4 float
     184          48 :         if (nNodeBytes < BLOB_HEADER_SIZE + SIZEOF_CELL * nCellCount)
     185             :         {
     186           0 :             CPLError(CE_Failure, CPLE_AppDefined,
     187             :                      "Not enough bytes when reading node " CPL_FRMT_GIB,
     188             :                      static_cast<GIntBig>(nNodeNo));
     189           0 :             return false;
     190             :         }
     191             : 
     192          48 :         size_t nOffset = BLOB_HEADER_SIZE;
     193          48 :         if (nCurDepth == nTreeDepth)
     194             :         {
     195             :             // Leaf node: it references feature IDs.
     196             : 
     197             :             // If we are about to go above m_nRowGroupSize, flush past
     198             :             // features now, to improve the spatial compacity of the row group.
     199          46 :             if (m_nRowGroupSize > nCellCount &&
     200          46 :                 nCountWrittenFeaturesSinceLastFlush + nCellCount >
     201          46 :                     m_nRowGroupSize)
     202             :             {
     203          14 :                 nCountWrittenFeaturesSinceLastFlush = 0;
     204          14 :                 if (!FlushFeatures())
     205             :                 {
     206           0 :                     return false;
     207             :                 }
     208             :             }
     209             : 
     210             :             // nCellCount shouldn't be over 51 normally, but even 65535
     211             :             // would be fine...
     212          46 :             assert(nCellCount <= 65535);
     213        1248 :             for (int i = 0; i < nCellCount; ++i)
     214             :             {
     215             :                 int64_t nFID;
     216        1202 :                 memcpy(&nFID, pabyNodeData + nOffset, sizeof(int64_t));
     217        1202 :                 CPL_MSBPTR64(&nFID);
     218             : 
     219             :                 const auto poSrcFeature = std::unique_ptr<const OGRFeature>(
     220        1202 :                     m_poTmpGPKGLayer->GetFeature(nFID));
     221        1202 :                 if (!poSrcFeature)
     222             :                 {
     223           0 :                     CPLError(CE_Failure, CPLE_AppDefined,
     224             :                              "Cannot get feature " CPL_FRMT_GIB,
     225             :                              static_cast<GIntBig>(nFID));
     226           0 :                     return false;
     227             :                 }
     228             : 
     229        1202 :                 int nBytesFeature = 0;
     230             :                 const GByte *pabyFeatureData =
     231        1202 :                     poSrcFeature->GetFieldAsBinary(0, &nBytesFeature);
     232        1202 :                 if (!oFeat.DeserializeFromBinary(pabyFeatureData,
     233             :                                                  nBytesFeature))
     234             :                 {
     235           0 :                     CPLError(CE_Failure, CPLE_AppDefined,
     236             :                              "Cannot deserialize feature");
     237           0 :                     return false;
     238             :                 }
     239        1202 :                 if (OGRArrowWriterLayer::ICreateFeature(&oFeat) != OGRERR_NONE)
     240             :                 {
     241           0 :                     return false;
     242             :                 }
     243             : 
     244        1202 :                 nOffset += SIZEOF_CELL;
     245             : 
     246        1202 :                 ++nCountWrittenFeaturesSinceLastFlush;
     247             : 
     248        1202 :                 if ((m_nFeatureCount % PROGRESS_FC_INTERVAL) == 0 ||
     249        1202 :                     m_nFeatureCount == m_nTmpFeatureCount / 2)
     250             :                 {
     251           2 :                     CPLDebugProgress(
     252             :                         "PARQUET",
     253             :                         "CopyTmpGpkgLayerToFinalFile(): %.02f%% progress",
     254           2 :                         100.0 * double(m_nFeatureCount) /
     255           2 :                             double(m_nTmpFeatureCount));
     256             :                 }
     257             :             }
     258             :         }
     259             :         else
     260             :         {
     261             :             // Non-leaf node: it references child nodes.
     262             : 
     263             :             // nCellCount shouldn't be over 51 normally, but even 65535
     264             :             // would be fine...
     265           2 :             assert(nCellCount <= 65535);
     266          48 :             for (int i = 0; i < nCellCount; ++i)
     267             :             {
     268             :                 int64_t nNode;
     269          46 :                 memcpy(&nNode, pabyNodeData + nOffset, sizeof(int64_t));
     270          46 :                 CPL_MSBPTR64(&nNode);
     271             :                 aNodeNoDepthPair.emplace_back(
     272          46 :                     std::make_pair(nNode, nCurDepth + 1));
     273          46 :                 nOffset += SIZEOF_CELL;
     274             :             }
     275             :         }
     276             :     }
     277             : 
     278           2 :     CPLDebug("PARQUET",
     279             :              "CopyTmpGpkgLayerToFinalFile(): 100%%, successfully finished");
     280           2 :     return true;
     281             : }
     282             : 
     283             : /************************************************************************/
     284             : /*                       IsSupportedGeometryType()                      */
     285             : /************************************************************************/
     286             : 
     287         271 : bool OGRParquetWriterLayer::IsSupportedGeometryType(
     288             :     OGRwkbGeometryType eGType) const
     289             : {
     290         271 :     const auto eFlattenType = wkbFlatten(eGType);
     291         271 :     if (!OGR_GT_HasM(eGType) && eFlattenType <= wkbGeometryCollection)
     292             :     {
     293         270 :         return true;
     294             :     }
     295             : 
     296             :     const auto osConfigOptionName =
     297           3 :         "OGR_" + GetDriverUCName() + "_ALLOW_ALL_DIMS";
     298           1 :     if (CPLTestBool(CPLGetConfigOption(osConfigOptionName.c_str(), "NO")))
     299             :     {
     300           0 :         return true;
     301             :     }
     302             : 
     303           1 :     CPLError(CE_Failure, CPLE_NotSupported,
     304             :              "Only 2D and Z geometry types are supported (unless the "
     305             :              "%s configuration option is set to YES)",
     306             :              osConfigOptionName.c_str());
     307           1 :     return false;
     308             : }
     309             : 
     310             : /************************************************************************/
     311             : /*                           SetOptions()                               */
     312             : /************************************************************************/
     313             : 
     314         272 : bool OGRParquetWriterLayer::SetOptions(CSLConstList papszOptions,
     315             :                                        const OGRSpatialReference *poSpatialRef,
     316             :                                        OGRwkbGeometryType eGType)
     317             : {
     318         272 :     const char *pszWriteCoveringBBox = CSLFetchNameValueDef(
     319             :         papszOptions, "WRITE_COVERING_BBOX",
     320             :         CPLGetConfigOption("OGR_PARQUET_WRITE_COVERING_BBOX", nullptr));
     321         272 :     m_bWriteBBoxStruct =
     322         272 :         pszWriteCoveringBBox == nullptr || CPLTestBool(pszWriteCoveringBBox);
     323             : 
     324         272 :     if (CPLTestBool(CSLFetchNameValueDef(papszOptions, "SORT_BY_BBOX", "NO")))
     325             :     {
     326           6 :         const std::string osTmpGPKG(std::string(m_poDataset->GetDescription()) +
     327           3 :                                     ".tmp.gpkg");
     328           3 :         auto poGPKGDrv = GetGDALDriverManager()->GetDriverByName("GPKG");
     329           3 :         if (!poGPKGDrv)
     330             :         {
     331           1 :             CPLError(
     332             :                 CE_Failure, CPLE_AppDefined,
     333             :                 "Driver GPKG required for SORT_BY_BBOX layer creation option");
     334           1 :             return false;
     335             :         }
     336           2 :         m_poTmpGPKG.reset(poGPKGDrv->Create(osTmpGPKG.c_str(), 0, 0, 0,
     337             :                                             GDT_Unknown, nullptr));
     338           2 :         if (!m_poTmpGPKG)
     339           0 :             return false;
     340           2 :         m_poTmpGPKG->MarkSuppressOnClose();
     341           2 :         m_poTmpGPKGLayer = m_poTmpGPKG->CreateLayer("tmp");
     342           2 :         if (!m_poTmpGPKGLayer)
     343           0 :             return false;
     344             :         // Serialized feature
     345           2 :         m_poTmpGPKGLayer->CreateField(
     346           2 :             std::make_unique<OGRFieldDefn>("serialized_feature", OFTBinary)
     347           2 :                 .get());
     348           2 :         CPL_IGNORE_RET_VAL(m_poTmpGPKGLayer->StartTransaction());
     349             :     }
     350             : 
     351             :     const char *pszGeomEncoding =
     352         271 :         CSLFetchNameValue(papszOptions, "GEOMETRY_ENCODING");
     353         271 :     m_eGeomEncoding = OGRArrowGeomEncoding::WKB;
     354         271 :     if (pszGeomEncoding)
     355             :     {
     356         148 :         if (EQUAL(pszGeomEncoding, "WKB"))
     357           0 :             m_eGeomEncoding = OGRArrowGeomEncoding::WKB;
     358         148 :         else if (EQUAL(pszGeomEncoding, "WKT"))
     359           8 :             m_eGeomEncoding = OGRArrowGeomEncoding::WKT;
     360         140 :         else if (EQUAL(pszGeomEncoding, "GEOARROW_INTERLEAVED"))
     361             :         {
     362          28 :             CPLErrorOnce(
     363             :                 CE_Warning, CPLE_AppDefined,
     364             :                 "Use of GEOMETRY_ENCODING=GEOARROW_INTERLEAVED is not "
     365             :                 "recommended. "
     366             :                 "GeoParquet 1.1 uses GEOMETRY_ENCODING=GEOARROW (struct) "
     367             :                 "instead.");
     368          28 :             m_eGeomEncoding = OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC;
     369             :         }
     370         112 :         else if (EQUAL(pszGeomEncoding, "GEOARROW") ||
     371           0 :                  EQUAL(pszGeomEncoding, "GEOARROW_STRUCT"))
     372         112 :             m_eGeomEncoding = OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC;
     373             :         else
     374             :         {
     375           0 :             CPLError(CE_Failure, CPLE_NotSupported,
     376             :                      "Unsupported GEOMETRY_ENCODING = %s", pszGeomEncoding);
     377           0 :             return false;
     378             :         }
     379             :     }
     380             : 
     381             :     const char *pszCoordPrecision =
     382         271 :         CSLFetchNameValue(papszOptions, "COORDINATE_PRECISION");
     383         271 :     if (pszCoordPrecision)
     384           0 :         m_nWKTCoordinatePrecision = atoi(pszCoordPrecision);
     385             : 
     386         271 :     m_bForceCounterClockwiseOrientation =
     387         271 :         EQUAL(CSLFetchNameValueDef(papszOptions, "POLYGON_ORIENTATION",
     388             :                                    "COUNTERCLOCKWISE"),
     389             :               "COUNTERCLOCKWISE");
     390             : 
     391         271 :     if (eGType != wkbNone)
     392             :     {
     393         245 :         if (!IsSupportedGeometryType(eGType))
     394             :         {
     395           1 :             return false;
     396             :         }
     397             : 
     398         244 :         m_poFeatureDefn->SetGeomType(eGType);
     399         244 :         auto eGeomEncoding = m_eGeomEncoding;
     400         244 :         if (eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC ||
     401         216 :             eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC)
     402             :         {
     403         140 :             const auto eEncodingType = eGeomEncoding;
     404         140 :             eGeomEncoding = GetPreciseArrowGeomEncoding(eEncodingType, eGType);
     405         140 :             if (eGeomEncoding == eEncodingType)
     406           0 :                 return false;
     407             :         }
     408         244 :         m_aeGeomEncoding.push_back(eGeomEncoding);
     409         244 :         m_poFeatureDefn->GetGeomFieldDefn(0)->SetName(
     410             :             CSLFetchNameValueDef(papszOptions, "GEOMETRY_NAME", "geometry"));
     411         244 :         if (poSpatialRef)
     412             :         {
     413          27 :             auto poSRS = poSpatialRef->Clone();
     414          27 :             m_poFeatureDefn->GetGeomFieldDefn(0)->SetSpatialRef(poSRS);
     415          27 :             poSRS->Release();
     416             :         }
     417             :     }
     418             : 
     419         270 :     m_osFIDColumn = CSLFetchNameValueDef(papszOptions, "FID", "");
     420             : 
     421         270 :     const char *pszCompression = CSLFetchNameValue(papszOptions, "COMPRESSION");
     422         270 :     if (pszCompression == nullptr)
     423             :     {
     424         792 :         auto oResult = arrow::util::Codec::GetCompressionType("snappy");
     425         264 :         if (oResult.ok() && arrow::util::Codec::IsAvailable(*oResult))
     426             :         {
     427         264 :             pszCompression = "SNAPPY";
     428             :         }
     429             :         else
     430             :         {
     431           0 :             pszCompression = "NONE";
     432             :         }
     433             :     }
     434             : 
     435         270 :     if (EQUAL(pszCompression, "NONE"))
     436           0 :         pszCompression = "UNCOMPRESSED";
     437             :     auto oResult = arrow::util::Codec::GetCompressionType(
     438         540 :         CPLString(pszCompression).tolower());
     439         270 :     if (!oResult.ok())
     440             :     {
     441           1 :         CPLError(CE_Failure, CPLE_NotSupported,
     442             :                  "Unrecognized compression method: %s", pszCompression);
     443           1 :         return false;
     444             :     }
     445         269 :     m_eCompression = *oResult;
     446         269 :     if (!arrow::util::Codec::IsAvailable(m_eCompression))
     447             :     {
     448           0 :         CPLError(CE_Failure, CPLE_NotSupported,
     449             :                  "Compression method %s is known, but libarrow has not "
     450             :                  "been built with support for it",
     451             :                  pszCompression);
     452           0 :         return false;
     453             :     }
     454         269 :     m_oWriterPropertiesBuilder.compression(m_eCompression);
     455             : 
     456             :     const char *pszCompressionLevel =
     457         269 :         CSLFetchNameValue(papszOptions, "COMPRESSION_LEVEL");
     458         269 :     if (pszCompressionLevel)
     459             :     {
     460           2 :         const int nCompressionLevel = atoi(pszCompressionLevel);
     461           2 :         if (nCompressionLevel != DEFAULT_COMPRESSION_LEVEL)
     462           2 :             m_oWriterPropertiesBuilder.compression_level(nCompressionLevel);
     463             :     }
     464             : 
     465             :     const std::string osCreator =
     466         269 :         CSLFetchNameValueDef(papszOptions, "CREATOR", "");
     467         269 :     if (!osCreator.empty())
     468           1 :         m_oWriterPropertiesBuilder.created_by(osCreator);
     469             :     else
     470         268 :         m_oWriterPropertiesBuilder.created_by("GDAL " GDAL_RELEASE_NAME
     471             :                                               ", using " CREATED_BY_VERSION);
     472             : 
     473             :     // Undocumented option. Not clear it is useful besides unit test purposes
     474         269 :     if (!CPLTestBool(CSLFetchNameValueDef(papszOptions, "STATISTICS", "YES")))
     475           1 :         m_oWriterPropertiesBuilder.disable_statistics();
     476             : 
     477             : #if PARQUET_VERSION_MAJOR >= 12
     478             :     // Undocumented option. Not clear it is useful to disable it.
     479         269 :     if (CPLTestBool(CSLFetchNameValueDef(papszOptions, "PAGE_INDEX", "YES")))
     480         269 :         m_oWriterPropertiesBuilder.enable_write_page_index();
     481             : #endif
     482             : 
     483             :     const char *pszWriteGeo =
     484         269 :         CPLGetConfigOption("OGR_PARQUET_WRITE_GEO", nullptr);
     485         269 :     m_bWriteGeoMetadata = pszWriteGeo == nullptr || CPLTestBool(pszWriteGeo);
     486             : 
     487         269 :     if (m_eGeomEncoding == OGRArrowGeomEncoding::WKB && eGType != wkbNone)
     488             :     {
     489             : #if ARROW_VERSION_MAJOR >= 21
     490             :         const char *pszUseParquetGeoTypes =
     491             :             CSLFetchNameValueDef(papszOptions, "USE_PARQUET_GEO_TYPES", "NO");
     492             :         if (EQUAL(pszUseParquetGeoTypes, "ONLY"))
     493             :         {
     494             :             m_bUseArrowWKBExtension = true;
     495             :             if (pszWriteGeo == nullptr)
     496             :                 m_bWriteGeoMetadata = false;
     497             :             if (pszWriteCoveringBBox == nullptr)
     498             :                 m_bWriteBBoxStruct = false;
     499             :         }
     500             :         else
     501             :         {
     502             :             m_bUseArrowWKBExtension = CPLTestBool(pszUseParquetGeoTypes);
     503             :         }
     504             : #else
     505         103 :         m_oWriterPropertiesBuilder.disable_statistics(
     506         309 :             parquet::schema::ColumnPath::FromDotString(
     507         103 :                 m_poFeatureDefn->GetGeomFieldDefn(0)->GetNameRef()));
     508             : #endif
     509             :     }
     510             : 
     511             :     const char *pszRowGroupSize =
     512         269 :         CSLFetchNameValue(papszOptions, "ROW_GROUP_SIZE");
     513         269 :     if (pszRowGroupSize)
     514             :     {
     515          10 :         auto nRowGroupSize = static_cast<int64_t>(atoll(pszRowGroupSize));
     516          10 :         if (nRowGroupSize > 0)
     517             :         {
     518          10 :             if (nRowGroupSize > INT_MAX)
     519           0 :                 nRowGroupSize = INT_MAX;
     520          10 :             m_nRowGroupSize = nRowGroupSize;
     521             :         }
     522             :     }
     523             : 
     524         269 :     m_bEdgesSpherical = EQUAL(
     525             :         CSLFetchNameValueDef(papszOptions, "EDGES", "PLANAR"), "SPHERICAL");
     526             : 
     527         269 :     m_bInitializationOK = true;
     528         269 :     return true;
     529             : }
     530             : 
     531             : /************************************************************************/
     532             : /*                         CloseFileWriter()                            */
     533             : /************************************************************************/
     534             : 
     535         269 : bool OGRParquetWriterLayer::CloseFileWriter()
     536             : {
     537         538 :     auto status = m_poFileWriter->Close();
     538         269 :     if (!status.ok())
     539             :     {
     540           0 :         CPLError(CE_Failure, CPLE_AppDefined,
     541             :                  "FileWriter::Close() failed with %s",
     542           0 :                  status.message().c_str());
     543             :     }
     544         538 :     return status.ok();
     545             : }
     546             : 
     547             : /************************************************************************/
     548             : /*                            GetGeoMetadata()                          */
     549             : /************************************************************************/
     550             : 
     551         269 : std::string OGRParquetWriterLayer::GetGeoMetadata() const
     552             : {
     553             :     // Just for unit testing purposes
     554             :     const char *pszGeoMetadata =
     555         269 :         CPLGetConfigOption("OGR_PARQUET_GEO_METADATA", nullptr);
     556         269 :     if (pszGeoMetadata)
     557          16 :         return pszGeoMetadata;
     558             : 
     559         253 :     if (m_poFeatureDefn->GetGeomFieldCount() != 0 && m_bWriteGeoMetadata)
     560             :     {
     561         472 :         CPLJSONObject oRoot;
     562         236 :         oRoot.Add("version", "1.1.0");
     563         236 :         oRoot.Add("primary_column",
     564         236 :                   m_poFeatureDefn->GetGeomFieldDefn(0)->GetNameRef());
     565         472 :         CPLJSONObject oColumns;
     566         236 :         oRoot.Add("columns", oColumns);
     567         489 :         for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
     568             :         {
     569         253 :             const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
     570         506 :             CPLJSONObject oColumn;
     571         253 :             oColumns.Add(poGeomFieldDefn->GetNameRef(), oColumn);
     572         253 :             oColumn.Add("encoding",
     573         253 :                         GetGeomEncodingAsString(m_aeGeomEncoding[i], true));
     574             : 
     575         253 :             if (CPLTestBool(CPLGetConfigOption("OGR_PARQUET_WRITE_CRS", "YES")))
     576             :             {
     577         252 :                 const auto poSRS = poGeomFieldDefn->GetSpatialRef();
     578         252 :                 if (poSRS)
     579             :                 {
     580          52 :                     OGRSpatialReference oSRSIdentified(IdentifyCRS(poSRS));
     581             : 
     582             :                     const char *pszAuthName =
     583          26 :                         oSRSIdentified.GetAuthorityName(nullptr);
     584             :                     const char *pszAuthCode =
     585          26 :                         oSRSIdentified.GetAuthorityCode(nullptr);
     586             : 
     587          26 :                     bool bOmitCRS = false;
     588          26 :                     if (pszAuthName != nullptr && pszAuthCode != nullptr &&
     589          25 :                         ((EQUAL(pszAuthName, "EPSG") &&
     590          22 :                           EQUAL(pszAuthCode, "4326")) ||
     591          14 :                          (EQUAL(pszAuthName, "OGC") &&
     592           3 :                           EQUAL(pszAuthCode, "CRS84"))))
     593             :                     {
     594             :                         // To make things less confusing for non-geo-aware
     595             :                         // consumers, omit EPSG:4326 / OGC:CRS84 CRS by default
     596          14 :                         bOmitCRS = CPLTestBool(CPLGetConfigOption(
     597             :                             "OGR_PARQUET_CRS_OMIT_IF_WGS84", "YES"));
     598             :                     }
     599             : 
     600          26 :                     if (bOmitCRS)
     601             :                     {
     602             :                         // do nothing
     603             :                     }
     604          12 :                     else if (EQUAL(CPLGetConfigOption(
     605             :                                        "OGR_PARQUET_CRS_ENCODING", "PROJJSON"),
     606             :                                    "PROJJSON"))
     607             :                     {
     608             :                         // CRS encoded as PROJJSON for GeoParquet >= 0.4.0
     609          12 :                         char *pszPROJJSON = nullptr;
     610          12 :                         oSRSIdentified.exportToPROJJSON(&pszPROJJSON, nullptr);
     611          24 :                         CPLJSONDocument oCRSDoc;
     612          12 :                         CPL_IGNORE_RET_VAL(oCRSDoc.LoadMemory(pszPROJJSON));
     613          12 :                         CPLFree(pszPROJJSON);
     614          12 :                         CPLJSONObject oCRSRoot = oCRSDoc.GetRoot();
     615          12 :                         RemoveIDFromMemberOfEnsembles(oCRSRoot);
     616          12 :                         oColumn.Add("crs", oCRSRoot);
     617             :                     }
     618             :                     else
     619             :                     {
     620             :                         // WKT was used in GeoParquet <= 0.3.0
     621           0 :                         const char *const apszOptions[] = {
     622             :                             "FORMAT=WKT2_2019", "MULTILINE=NO", nullptr};
     623           0 :                         char *pszWKT = nullptr;
     624           0 :                         oSRSIdentified.exportToWkt(&pszWKT, apszOptions);
     625           0 :                         if (pszWKT)
     626           0 :                             oColumn.Add("crs", pszWKT);
     627           0 :                         CPLFree(pszWKT);
     628             :                     }
     629             : 
     630          26 :                     const double dfCoordEpoch = poSRS->GetCoordinateEpoch();
     631          26 :                     if (dfCoordEpoch > 0)
     632           2 :                         oColumn.Add("epoch", dfCoordEpoch);
     633             :                 }
     634             :                 else
     635             :                 {
     636         226 :                     oColumn.AddNull("crs");
     637             :                 }
     638             :             }
     639             : 
     640         253 :             if (m_bEdgesSpherical)
     641             :             {
     642           1 :                 oColumn.Add("edges", "spherical");
     643             :             }
     644             : 
     645         479 :             if (m_aoEnvelopes[i].IsInit() &&
     646         226 :                 CPLTestBool(
     647             :                     CPLGetConfigOption("OGR_PARQUET_WRITE_BBOX", "YES")))
     648             :             {
     649         226 :                 bool bHasZ = false;
     650         411 :                 for (const auto eGeomType : m_oSetWrittenGeometryTypes[i])
     651             :                 {
     652         268 :                     bHasZ = OGR_GT_HasZ(eGeomType);
     653         268 :                     if (bHasZ)
     654          83 :                         break;
     655             :                 }
     656         226 :                 CPLJSONArray oBBOX;
     657         226 :                 oBBOX.Add(m_aoEnvelopes[i].MinX);
     658         226 :                 oBBOX.Add(m_aoEnvelopes[i].MinY);
     659         226 :                 if (bHasZ)
     660          83 :                     oBBOX.Add(m_aoEnvelopes[i].MinZ);
     661         226 :                 oBBOX.Add(m_aoEnvelopes[i].MaxX);
     662         226 :                 oBBOX.Add(m_aoEnvelopes[i].MaxY);
     663         226 :                 if (bHasZ)
     664          83 :                     oBBOX.Add(m_aoEnvelopes[i].MaxZ);
     665         226 :                 oColumn.Add("bbox", oBBOX);
     666             :             }
     667             : 
     668             :             // Bounding box column definition
     669         436 :             if (m_bWriteBBoxStruct &&
     670         183 :                 CPLTestBool(CPLGetConfigOption(
     671             :                     "OGR_PARQUET_WRITE_COVERING_BBOX_IN_METADATA", "YES")))
     672             :             {
     673         366 :                 CPLJSONObject oCovering;
     674         183 :                 oColumn.Add("covering", oCovering);
     675         366 :                 CPLJSONObject oBBOX;
     676         183 :                 oCovering.Add("bbox", oBBOX);
     677             :                 const auto AddComponent =
     678        2196 :                     [this, i, &oBBOX](const char *pszComponent)
     679             :                 {
     680         732 :                     CPLJSONArray oArray;
     681         732 :                     oArray.Add(m_apoFieldsBBOX[i]->name());
     682         732 :                     oArray.Add(pszComponent);
     683         732 :                     oBBOX.Add(pszComponent, oArray);
     684         732 :                 };
     685         183 :                 AddComponent("xmin");
     686         183 :                 AddComponent("ymin");
     687         183 :                 AddComponent("xmax");
     688         183 :                 AddComponent("ymax");
     689             :             }
     690             : 
     691         284 :             const auto GetStringGeometryType = [](OGRwkbGeometryType eType)
     692             :             {
     693         284 :                 const auto eFlattenType = wkbFlatten(eType);
     694         284 :                 std::string osType = "Unknown";
     695         284 :                 if (wkbPoint == eFlattenType)
     696          66 :                     osType = "Point";
     697         218 :                 else if (wkbLineString == eFlattenType)
     698          34 :                     osType = "LineString";
     699         184 :                 else if (wkbPolygon == eFlattenType)
     700          55 :                     osType = "Polygon";
     701         129 :                 else if (wkbMultiPoint == eFlattenType)
     702          26 :                     osType = "MultiPoint";
     703         103 :                 else if (wkbMultiLineString == eFlattenType)
     704          29 :                     osType = "MultiLineString";
     705          74 :                 else if (wkbMultiPolygon == eFlattenType)
     706          69 :                     osType = "MultiPolygon";
     707           5 :                 else if (wkbGeometryCollection == eFlattenType)
     708           5 :                     osType = "GeometryCollection";
     709         284 :                 if (osType != "Unknown")
     710             :                 {
     711             :                     // M and ZM not supported officially currently, but it
     712             :                     // doesn't hurt to anticipate
     713         284 :                     if (OGR_GT_HasZ(eType) && OGR_GT_HasM(eType))
     714           8 :                         osType += " ZM";
     715         276 :                     else if (OGR_GT_HasZ(eType))
     716          91 :                         osType += " Z";
     717         185 :                     else if (OGR_GT_HasM(eType))
     718           8 :                         osType += " M";
     719             :                 }
     720         284 :                 return osType;
     721             :             };
     722             : 
     723         253 :             if (m_bForceCounterClockwiseOrientation)
     724         252 :                 oColumn.Add("orientation", "counterclockwise");
     725             : 
     726         253 :             CPLJSONArray oArray;
     727         537 :             for (const auto eType : m_oSetWrittenGeometryTypes[i])
     728             :             {
     729         284 :                 oArray.Add(GetStringGeometryType(eType));
     730             :             }
     731         253 :             oColumn.Add("geometry_types", oArray);
     732             :         }
     733             : 
     734         236 :         return oRoot.Format(CPLJSONObject::PrettyFormat::Plain);
     735             :     }
     736          17 :     return std::string();
     737             : }
     738             : 
     739             : /************************************************************************/
     740             : /*               PerformStepsBeforeFinalFlushGroup()                    */
     741             : /************************************************************************/
     742             : 
     743         269 : void OGRParquetWriterLayer::PerformStepsBeforeFinalFlushGroup()
     744             : {
     745         269 :     if (m_poKeyValueMetadata)
     746             :     {
     747         538 :         std::string osGeoMetadata = GetGeoMetadata();
     748         538 :         auto poTmpSchema = m_poSchema;
     749         269 :         if (!osGeoMetadata.empty())
     750             :         {
     751             :             // HACK: it would be good for Arrow to provide a clean way to alter
     752             :             // key value metadata before finalizing.
     753             :             // We need to write metadata at end to write the bounding box.
     754         252 :             const_cast<arrow::KeyValueMetadata *>(m_poKeyValueMetadata.get())
     755         252 :                 ->Append("geo", osGeoMetadata);
     756             : 
     757         252 :             auto kvMetadata = poTmpSchema->metadata()
     758          11 :                                   ? poTmpSchema->metadata()->Copy()
     759         263 :                                   : std::make_shared<arrow::KeyValueMetadata>();
     760         252 :             kvMetadata->Append("geo", std::move(osGeoMetadata));
     761         252 :             poTmpSchema = poTmpSchema->WithMetadata(kvMetadata);
     762             :         }
     763             : 
     764         269 :         if (CPLTestBool(
     765             :                 CPLGetConfigOption("OGR_PARQUET_WRITE_ARROW_SCHEMA", "YES")))
     766             :         {
     767             :             auto status =
     768         538 :                 ::arrow::ipc::SerializeSchema(*poTmpSchema, m_poMemoryPool);
     769         269 :             if (status.ok())
     770             :             {
     771             :                 // The serialized schema is not UTF-8, which is required for
     772             :                 // Thrift
     773         538 :                 const std::string schema_as_string = (*status)->ToString();
     774             :                 std::string schema_base64 =
     775         269 :                     ::arrow::util::base64_encode(schema_as_string);
     776         269 :                 static const std::string kArrowSchemaKey = "ARROW:schema";
     777             :                 const_cast<arrow::KeyValueMetadata *>(
     778         269 :                     m_poKeyValueMetadata.get())
     779         269 :                     ->Append(kArrowSchemaKey, std::move(schema_base64));
     780             :             }
     781             :         }
     782             : 
     783             :         // Put GDAL metadata into a gdal:metadata domain
     784         538 :         CPLJSONObject oMultiMetadata;
     785         269 :         bool bHasMultiMetadata = false;
     786         275 :         auto &l_oMDMD = oMDMD.GetDomainList() && *(oMDMD.GetDomainList())
     787         275 :                             ? oMDMD
     788         263 :                             : m_poDataset->GetMultiDomainMetadata();
     789         277 :         for (CSLConstList papszDomainIter = l_oMDMD.GetDomainList();
     790         277 :              papszDomainIter && *papszDomainIter; ++papszDomainIter)
     791             :         {
     792           8 :             const char *pszDomain = *papszDomainIter;
     793           8 :             CSLConstList papszMD = l_oMDMD.GetMetadata(pszDomain);
     794           8 :             if (STARTS_WITH(pszDomain, "json:") && papszMD && papszMD[0])
     795             :             {
     796           1 :                 CPLJSONDocument oDoc;
     797           1 :                 if (oDoc.LoadMemory(papszMD[0]))
     798             :                 {
     799           1 :                     bHasMultiMetadata = true;
     800           1 :                     oMultiMetadata.Add(pszDomain, oDoc.GetRoot());
     801           1 :                     continue;
     802           0 :                 }
     803             :             }
     804           7 :             else if (STARTS_WITH(pszDomain, "xml:") && papszMD && papszMD[0])
     805             :             {
     806           1 :                 bHasMultiMetadata = true;
     807           1 :                 oMultiMetadata.Add(pszDomain, papszMD[0]);
     808           1 :                 continue;
     809             :             }
     810          12 :             CPLJSONObject oMetadata;
     811           6 :             bool bHasMetadata = false;
     812          12 :             for (CSLConstList papszMDIter = papszMD;
     813          12 :                  papszMDIter && *papszMDIter; ++papszMDIter)
     814             :             {
     815           6 :                 char *pszKey = nullptr;
     816           6 :                 const char *pszValue = CPLParseNameValue(*papszMDIter, &pszKey);
     817           6 :                 if (pszKey && pszValue)
     818             :                 {
     819           6 :                     bHasMetadata = true;
     820           6 :                     bHasMultiMetadata = true;
     821           6 :                     oMetadata.Add(pszKey, pszValue);
     822             :                 }
     823           6 :                 CPLFree(pszKey);
     824             :             }
     825           6 :             if (bHasMetadata)
     826           6 :                 oMultiMetadata.Add(pszDomain, oMetadata);
     827             :         }
     828         269 :         if (bHasMultiMetadata)
     829             :         {
     830           6 :             const_cast<arrow::KeyValueMetadata *>(m_poKeyValueMetadata.get())
     831           6 :                 ->Append(
     832             :                     "gdal:metadata",
     833          12 :                     oMultiMetadata.Format(CPLJSONObject::PrettyFormat::Plain));
     834             :         }
     835             :     }
     836         269 : }
     837             : 
     838             : /************************************************************************/
     839             : /*                                 Open()                               */
     840             : /************************************************************************/
     841             : 
     842             : // Same as parquet::arrow::FileWriter::Open(), except we also
     843             : // return KeyValueMetadata
     844             : static arrow::Status
     845         269 : Open(const ::arrow::Schema &schema, ::arrow::MemoryPool *pool,
     846             :      std::shared_ptr<::arrow::io::OutputStream> sink,
     847             :      std::shared_ptr<parquet::WriterProperties> properties,
     848             :      std::shared_ptr<parquet::ArrowWriterProperties> arrow_properties,
     849             :      std::unique_ptr<parquet::arrow::FileWriter> *writer,
     850             :      std::shared_ptr<const arrow::KeyValueMetadata> *outMetadata)
     851             : {
     852         269 :     std::shared_ptr<parquet::SchemaDescriptor> parquet_schema;
     853         538 :     RETURN_NOT_OK(parquet::arrow::ToParquetSchema(
     854             :         &schema, *properties, *arrow_properties, &parquet_schema));
     855             : 
     856             :     auto schema_node = std::static_pointer_cast<parquet::schema::GroupNode>(
     857         538 :         parquet_schema->schema_root());
     858             : 
     859         269 :     auto metadata = schema.metadata()
     860          16 :                         ? schema.metadata()->Copy()
     861         554 :                         : std::make_shared<arrow::KeyValueMetadata>();
     862         269 :     *outMetadata = metadata;
     863             : 
     864         269 :     std::unique_ptr<parquet::ParquetFileWriter> base_writer;
     865         269 :     PARQUET_CATCH_NOT_OK(base_writer = parquet::ParquetFileWriter::Open(
     866             :                              std::move(sink), std::move(schema_node),
     867             :                              std::move(properties), metadata));
     868             : 
     869         269 :     auto schema_ptr = std::make_shared<::arrow::Schema>(schema);
     870             :     return parquet::arrow::FileWriter::Make(
     871         538 :         pool, std::move(base_writer), std::move(schema_ptr),
     872         807 :         std::move(arrow_properties), writer);
     873             : }
     874             : 
     875             : /************************************************************************/
     876             : /*                          CreateSchema()                              */
     877             : /************************************************************************/
     878             : 
     879         269 : void OGRParquetWriterLayer::CreateSchema()
     880             : {
     881         269 :     CreateSchemaCommon();
     882         269 : }
     883             : 
     884             : /************************************************************************/
     885             : /*                          CreateGeomField()                           */
     886             : /************************************************************************/
     887             : 
     888          27 : OGRErr OGRParquetWriterLayer::CreateGeomField(const OGRGeomFieldDefn *poField,
     889             :                                               int bApproxOK)
     890             : {
     891          27 :     OGRErr eErr = OGRArrowWriterLayer::CreateGeomField(poField, bApproxOK);
     892          53 :     if (eErr == OGRERR_NONE &&
     893          26 :         m_aeGeomEncoding.back() == OGRArrowGeomEncoding::WKB
     894             : #if ARROW_VERSION_MAJOR < 21
     895             :         // Geostatistics in Arrow 21 do not support geographic type for now
     896          53 :         && m_bEdgesSpherical
     897             : #endif
     898             :     )
     899             :     {
     900           0 :         m_oWriterPropertiesBuilder.disable_statistics(
     901           0 :             parquet::schema::ColumnPath::FromDotString(
     902           0 :                 m_poFeatureDefn
     903           0 :                     ->GetGeomFieldDefn(m_poFeatureDefn->GetGeomFieldCount() - 1)
     904             :                     ->GetNameRef()));
     905             :     }
     906          27 :     return eErr;
     907             : }
     908             : 
     909             : /************************************************************************/
     910             : /*                          CreateWriter()                              */
     911             : /************************************************************************/
     912             : 
     913         269 : void OGRParquetWriterLayer::CreateWriter()
     914             : {
     915         269 :     CPLAssert(m_poFileWriter == nullptr);
     916             : 
     917         269 :     if (m_poSchema == nullptr)
     918             :     {
     919          40 :         CreateSchema();
     920             :     }
     921             :     else
     922             :     {
     923         229 :         FinalizeSchema();
     924             :     }
     925             : 
     926             :     auto arrowWriterProperties =
     927         269 :         parquet::ArrowWriterProperties::Builder().store_schema()->build();
     928         807 :     CPL_IGNORE_RET_VAL(Open(*m_poSchema, m_poMemoryPool, m_poOutputStream,
     929         538 :                             m_oWriterPropertiesBuilder.build(),
     930         269 :                             std::move(arrowWriterProperties), &m_poFileWriter,
     931             :                             &m_poKeyValueMetadata));
     932         269 : }
     933             : 
     934             : /************************************************************************/
     935             : /*                          ICreateFeature()                            */
     936             : /************************************************************************/
     937             : 
     938        3091 : OGRErr OGRParquetWriterLayer::ICreateFeature(OGRFeature *poFeature)
     939             : {
     940             :     // If not using SORT_BY_BBOX=YES layer creation option, we can directly
     941             :     // write features to the final Parquet file
     942        3091 :     if (!m_poTmpGPKGLayer)
     943         887 :         return OGRArrowWriterLayer::ICreateFeature(poFeature);
     944             : 
     945             :     // SORT_BY_BBOX=YES case: we write for now a serialized version of poFeature
     946             :     // in a temporary GeoPackage file.
     947             : 
     948        2204 :     GIntBig nFID = poFeature->GetFID();
     949        2204 :     if (!m_osFIDColumn.empty() && nFID == OGRNullFID)
     950             :     {
     951        1102 :         nFID = m_nTmpFeatureCount;
     952        1102 :         poFeature->SetFID(nFID);
     953             :     }
     954        2204 :     ++m_nTmpFeatureCount;
     955             : 
     956        4408 :     std::vector<GByte> abyBuffer;
     957             :     // Serialize the source feature as a single array of bytes to preserve it
     958             :     // fully
     959        2204 :     if (!poFeature->SerializeToBinary(abyBuffer))
     960             :     {
     961           0 :         return OGRERR_FAILURE;
     962             :     }
     963             : 
     964             :     // SQLite3 limitation: a row must fit in slightly less than 1 GB.
     965        2204 :     constexpr int SOME_MARGIN = 128;
     966        2204 :     if (abyBuffer.size() > 1024 * 1024 * 1024 - SOME_MARGIN)
     967             :     {
     968           0 :         CPLError(CE_Failure, CPLE_NotSupported,
     969             :                  "Features larger than 1 GB are not supported");
     970           0 :         return OGRERR_FAILURE;
     971             :     }
     972             : 
     973        4408 :     OGRFeature oFeat(m_poTmpGPKGLayer->GetLayerDefn());
     974        2204 :     oFeat.SetFID(nFID);
     975        2204 :     oFeat.SetField(0, static_cast<int>(abyBuffer.size()), abyBuffer.data());
     976        2204 :     const auto poSrcGeom = poFeature->GetGeometryRef();
     977        2204 :     if (poSrcGeom && !poSrcGeom->IsEmpty())
     978             :     {
     979             :         // For the purpose of building an RTree, just use the bounding box of
     980             :         // the geometry as the geometry.
     981        1202 :         OGREnvelope sEnvelope;
     982        1202 :         poSrcGeom->getEnvelope(&sEnvelope);
     983        2404 :         auto poPoly = std::make_unique<OGRPolygon>();
     984        2404 :         auto poLR = std::make_unique<OGRLinearRing>();
     985        1202 :         poLR->addPoint(sEnvelope.MinX, sEnvelope.MinY);
     986        1202 :         poLR->addPoint(sEnvelope.MinX, sEnvelope.MaxY);
     987        1202 :         poLR->addPoint(sEnvelope.MaxX, sEnvelope.MaxY);
     988        1202 :         poLR->addPoint(sEnvelope.MaxX, sEnvelope.MinY);
     989        1202 :         poLR->addPoint(sEnvelope.MinX, sEnvelope.MinY);
     990        1202 :         poPoly->addRingDirectly(poLR.release());
     991        1202 :         oFeat.SetGeometryDirectly(poPoly.release());
     992             :     }
     993        2204 :     return m_poTmpGPKGLayer->CreateFeature(&oFeat);
     994             : }
     995             : 
     996             : /************************************************************************/
     997             : /*                            FlushGroup()                              */
     998             : /************************************************************************/
     999             : 
    1000         253 : bool OGRParquetWriterLayer::FlushGroup()
    1001             : {
    1002             : #if PARQUET_VERSION_MAJOR >= 20
    1003             :     auto status = m_poFileWriter->NewRowGroup();
    1004             : #else
    1005         506 :     auto status = m_poFileWriter->NewRowGroup(m_apoBuilders[0]->length());
    1006             : #endif
    1007         253 :     if (!status.ok())
    1008             :     {
    1009           0 :         CPLError(CE_Failure, CPLE_AppDefined, "NewRowGroup() failed with %s",
    1010           0 :                  status.message().c_str());
    1011           0 :         ClearArrayBuilers();
    1012           0 :         return false;
    1013             :     }
    1014             : 
    1015         253 :     auto ret = WriteArrays(
    1016        1025 :         [this](const std::shared_ptr<arrow::Field> &field,
    1017        1025 :                const std::shared_ptr<arrow::Array> &array)
    1018             :         {
    1019        2050 :             auto l_status = m_poFileWriter->WriteColumnChunk(*array);
    1020        1025 :             if (!l_status.ok())
    1021             :             {
    1022           0 :                 CPLError(CE_Failure, CPLE_AppDefined,
    1023             :                          "WriteColumnChunk() failed for field %s: %s",
    1024           0 :                          field->name().c_str(), l_status.message().c_str());
    1025           0 :                 return false;
    1026             :             }
    1027        1025 :             return true;
    1028             :         });
    1029             : 
    1030         253 :     ClearArrayBuilers();
    1031         253 :     return ret;
    1032             : }
    1033             : 
    1034             : /************************************************************************/
    1035             : /*                    FixupWKBGeometryBeforeWriting()                   */
    1036             : /************************************************************************/
    1037             : 
    1038          43 : void OGRParquetWriterLayer::FixupWKBGeometryBeforeWriting(GByte *pabyWkb,
    1039             :                                                           size_t nLen)
    1040             : {
    1041          43 :     if (!m_bForceCounterClockwiseOrientation)
    1042           0 :         return;
    1043             : 
    1044          43 :     OGRWKBFixupCounterClockWiseExternalRing(pabyWkb, nLen);
    1045             : }
    1046             : 
    1047             : /************************************************************************/
    1048             : /*                     FixupGeometryBeforeWriting()                     */
    1049             : /************************************************************************/
    1050             : 
    1051        1354 : void OGRParquetWriterLayer::FixupGeometryBeforeWriting(OGRGeometry *poGeom)
    1052             : {
    1053        1354 :     if (!m_bForceCounterClockwiseOrientation)
    1054           3 :         return;
    1055             : 
    1056        1351 :     const auto eFlattenType = wkbFlatten(poGeom->getGeometryType());
    1057             :     // Polygon rings MUST follow the right-hand rule for orientation
    1058             :     // (counterclockwise external rings, clockwise internal rings)
    1059        1351 :     if (eFlattenType == wkbPolygon)
    1060             :     {
    1061          64 :         bool bFirstRing = true;
    1062         131 :         for (auto poRing : poGeom->toPolygon())
    1063             :         {
    1064          75 :             if ((bFirstRing && poRing->isClockwise()) ||
    1065           8 :                 (!bFirstRing && !poRing->isClockwise()))
    1066             :             {
    1067          62 :                 poRing->reversePoints();
    1068             :             }
    1069          67 :             bFirstRing = false;
    1070             :         }
    1071             :     }
    1072        1287 :     else if (eFlattenType == wkbMultiPolygon ||
    1073             :              eFlattenType == wkbGeometryCollection)
    1074             :     {
    1075          35 :         for (auto poSubGeom : poGeom->toGeometryCollection())
    1076             :         {
    1077          21 :             FixupGeometryBeforeWriting(poSubGeom);
    1078             :         }
    1079             :     }
    1080             : }
    1081             : 
    1082             : /************************************************************************/
    1083             : /*                          WriteArrowBatch()                           */
    1084             : /************************************************************************/
    1085             : 
    1086             : #if PARQUET_VERSION_MAJOR > 10
    1087             : inline bool
    1088          14 : OGRParquetWriterLayer::WriteArrowBatch(const struct ArrowSchema *schema,
    1089             :                                        struct ArrowArray *array,
    1090             :                                        CSLConstList papszOptions)
    1091             : {
    1092          14 :     if (m_poTmpGPKGLayer)
    1093             :     {
    1094             :         // When using SORT_BY_BBOX=YES option, we can't directly write the
    1095             :         // input array, because we need to sort features. Hence we fallback
    1096             :         // to the OGRLayer base implementation, which will ultimately call
    1097             :         // OGRParquetWriterLayer::ICreateFeature()
    1098           0 :         return OGRLayer::WriteArrowBatch(schema, array, papszOptions);
    1099             :     }
    1100             : 
    1101          28 :     return WriteArrowBatchInternal(
    1102             :         schema, array, papszOptions,
    1103          28 :         [this](const std::shared_ptr<arrow::RecordBatch> &poBatch)
    1104             :         {
    1105          28 :             auto status = m_poFileWriter->NewBufferedRowGroup();
    1106          14 :             if (!status.ok())
    1107             :             {
    1108           0 :                 CPLError(CE_Failure, CPLE_AppDefined,
    1109             :                          "NewBufferedRowGroup() failed with %s",
    1110           0 :                          status.message().c_str());
    1111           0 :                 return false;
    1112             :             }
    1113             : 
    1114          14 :             status = m_poFileWriter->WriteRecordBatch(*poBatch);
    1115          14 :             if (!status.ok())
    1116             :             {
    1117           0 :                 CPLError(CE_Failure, CPLE_AppDefined,
    1118             :                          "WriteRecordBatch() failed: %s",
    1119           0 :                          status.message().c_str());
    1120           0 :                 return false;
    1121             :             }
    1122             : 
    1123          14 :             return true;
    1124          14 :         });
    1125             : }
    1126             : #endif
    1127             : 
    1128             : /************************************************************************/
    1129             : /*                         TestCapability()                             */
    1130             : /************************************************************************/
    1131             : 
    1132         491 : inline int OGRParquetWriterLayer::TestCapability(const char *pszCap)
    1133             : {
    1134             : #if PARQUET_VERSION_MAJOR <= 10
    1135             :     if (EQUAL(pszCap, OLCFastWriteArrowBatch))
    1136             :         return false;
    1137             : #endif
    1138             : 
    1139         491 :     if (m_poTmpGPKGLayer && EQUAL(pszCap, OLCFastWriteArrowBatch))
    1140             :     {
    1141             :         // When using SORT_BY_BBOX=YES option, we can't directly write the
    1142             :         // input array, because we need to sort features. So this is not
    1143             :         // fast
    1144           1 :         return false;
    1145             :     }
    1146             : 
    1147         490 :     return OGRArrowWriterLayer::TestCapability(pszCap);
    1148             : }
    1149             : 
    1150             : /************************************************************************/
    1151             : /*                        CreateFieldFromArrowSchema()                  */
    1152             : /************************************************************************/
    1153             : 
    1154             : #if PARQUET_VERSION_MAJOR > 10
    1155         396 : bool OGRParquetWriterLayer::CreateFieldFromArrowSchema(
    1156             :     const struct ArrowSchema *schema, CSLConstList papszOptions)
    1157             : {
    1158         396 :     if (m_poTmpGPKGLayer)
    1159             :     {
    1160             :         // When using SORT_BY_BBOX=YES option, we can't directly write the
    1161             :         // input array, because we need to sort features. But this process
    1162             :         // only supports the base Arrow types supported by
    1163             :         // OGRLayer::WriteArrowBatch()
    1164           0 :         return OGRLayer::CreateFieldFromArrowSchema(schema, papszOptions);
    1165             :     }
    1166             : 
    1167         396 :     return OGRArrowWriterLayer::CreateFieldFromArrowSchema(schema,
    1168         396 :                                                            papszOptions);
    1169             : }
    1170             : #endif
    1171             : 
    1172             : /************************************************************************/
    1173             : /*                        IsArrowSchemaSupported()                      */
    1174             : /************************************************************************/
    1175             : 
    1176             : #if PARQUET_VERSION_MAJOR > 10
    1177        1077 : bool OGRParquetWriterLayer::IsArrowSchemaSupported(
    1178             :     const struct ArrowSchema *schema, CSLConstList papszOptions,
    1179             :     std::string &osErrorMsg) const
    1180             : {
    1181        1077 :     if (m_poTmpGPKGLayer)
    1182             :     {
    1183             :         // When using SORT_BY_BBOX=YES option, we can't directly write the
    1184             :         // input array, because we need to sort features. But this process
    1185             :         // only supports the base Arrow types supported by
    1186             :         // OGRLayer::WriteArrowBatch()
    1187           0 :         return OGRLayer::IsArrowSchemaSupported(schema, papszOptions,
    1188           0 :                                                 osErrorMsg);
    1189             :     }
    1190             : 
    1191        1077 :     if (schema->format[0] == 'e' && schema->format[1] == 0)
    1192             :     {
    1193           1 :         osErrorMsg = "float16 not supported";
    1194           1 :         return false;
    1195             :     }
    1196        1076 :     if (schema->format[0] == 'v' && schema->format[1] == 'u')
    1197             :     {
    1198           1 :         osErrorMsg = "StringView not supported";
    1199           1 :         return false;
    1200             :     }
    1201        1075 :     if (schema->format[0] == 'v' && schema->format[1] == 'z')
    1202             :     {
    1203           1 :         osErrorMsg = "BinaryView not supported";
    1204           1 :         return false;
    1205             :     }
    1206        1074 :     if (schema->format[0] == '+' && schema->format[1] == 'v')
    1207             :     {
    1208           0 :         if (schema->format[2] == 'l')
    1209             :         {
    1210           0 :             osErrorMsg = "ListView not supported";
    1211           0 :             return false;
    1212             :         }
    1213           0 :         else if (schema->format[2] == 'L')
    1214             :         {
    1215           0 :             osErrorMsg = "LargeListView not supported";
    1216           0 :             return false;
    1217             :         }
    1218             :     }
    1219        2136 :     for (int64_t i = 0; i < schema->n_children; ++i)
    1220             :     {
    1221        1065 :         if (!IsArrowSchemaSupported(schema->children[i], papszOptions,
    1222             :                                     osErrorMsg))
    1223             :         {
    1224           3 :             return false;
    1225             :         }
    1226             :     }
    1227        1071 :     return true;
    1228             : }
    1229             : #endif
    1230             : 
    1231             : /************************************************************************/
    1232             : /*                            SetMetadata()                             */
    1233             : /************************************************************************/
    1234             : 
    1235          11 : CPLErr OGRParquetWriterLayer::SetMetadata(char **papszMetadata,
    1236             :                                           const char *pszDomain)
    1237             : {
    1238          11 :     if (!pszDomain || !EQUAL(pszDomain, "SHAPEFILE"))
    1239             :     {
    1240           7 :         return OGRLayer::SetMetadata(papszMetadata, pszDomain);
    1241             :     }
    1242           4 :     return CE_None;
    1243             : }
    1244             : 
    1245             : /************************************************************************/
    1246             : /*                             GetDataset()                             */
    1247             : /************************************************************************/
    1248             : 
    1249          23 : GDALDataset *OGRParquetWriterLayer::GetDataset()
    1250             : {
    1251          23 :     return m_poDataset;
    1252             : }