Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: Parquet Translator
4 : * Purpose: Implements OGRParquetDriver.
5 : * Author: Even Rouault, <even.rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2022, Planet Labs
9 : *
10 : * SPDX-License-Identifier: MIT
11 : ****************************************************************************/
12 :
13 : #ifdef STANDALONE
14 : #include "gdal_version.h"
15 : #else
16 : #undef DO_NOT_DEFINE_GDAL_DATE_NAME
17 : #include "gdal_version_full/gdal_version.h"
18 : #endif
19 :
20 : #include "ogr_parquet.h"
21 :
22 : #include "../arrow_common/ograrrowwriterlayer.hpp"
23 :
24 : #include "ogr_wkb.h"
25 :
26 : #include <cassert>
27 : #include <utility>
28 :
29 : /************************************************************************/
30 : /* OGRParquetWriterLayer() */
31 : /************************************************************************/
32 :
33 293 : OGRParquetWriterLayer::OGRParquetWriterLayer(
34 : OGRParquetWriterDataset *poDataset, arrow::MemoryPool *poMemoryPool,
35 : const std::shared_ptr<arrow::io::OutputStream> &poOutputStream,
36 293 : const char *pszLayerName)
37 : : OGRArrowWriterLayer(poMemoryPool, poOutputStream, pszLayerName),
38 293 : m_poDataset(poDataset)
39 : {
40 293 : m_bWriteFieldArrowExtensionName = CPLTestBool(
41 : CPLGetConfigOption("OGR_PARQUET_WRITE_ARROW_EXTENSION_NAME", "NO"));
42 293 : }
43 :
44 : /************************************************************************/
45 : /* Close() */
46 : /************************************************************************/
47 :
48 290 : bool OGRParquetWriterLayer::Close()
49 : {
50 290 : if (m_poTmpGPKGLayer)
51 : {
52 3 : if (!CopyTmpGpkgLayerToFinalFile())
53 0 : return false;
54 : }
55 :
56 290 : if (m_bInitializationOK)
57 : {
58 290 : if (!FinalizeWriting())
59 0 : return false;
60 : }
61 :
62 290 : return true;
63 : }
64 :
65 : /************************************************************************/
66 : /* CopyTmpGpkgLayerToFinalFile() */
67 : /************************************************************************/
68 :
69 3 : bool OGRParquetWriterLayer::CopyTmpGpkgLayerToFinalFile()
70 : {
71 3 : if (!m_poTmpGPKGLayer)
72 : {
73 0 : return true;
74 : }
75 :
76 3 : CPLDebug("PARQUET", "CopyTmpGpkgLayerToFinalFile(): start...");
77 :
78 3 : VSIUnlink(m_poTmpGPKG->GetDescription());
79 :
80 6 : OGRFeature oFeat(m_poFeatureDefn);
81 :
82 : // Interval in terms of features between 2 debug progress report messages
83 3 : constexpr int PROGRESS_FC_INTERVAL = 100 * 1000;
84 :
85 : // First, write features without geometries
86 : {
87 3 : auto poTmpLayer = std::unique_ptr<OGRLayer>(m_poTmpGPKG->ExecuteSQL(
88 : "SELECT serialized_feature FROM tmp WHERE fid NOT IN (SELECT id "
89 : "FROM rtree_tmp_geom)",
90 3 : nullptr, nullptr));
91 3 : if (!poTmpLayer)
92 0 : return false;
93 1005 : for (const auto &poSrcFeature : poTmpLayer.get())
94 : {
95 1002 : int nBytesFeature = 0;
96 : const GByte *pabyFeatureData =
97 1002 : poSrcFeature->GetFieldAsBinary(0, &nBytesFeature);
98 1002 : if (!oFeat.DeserializeFromBinary(pabyFeatureData, nBytesFeature))
99 : {
100 0 : CPLError(CE_Failure, CPLE_AppDefined,
101 : "Cannot deserialize feature");
102 0 : return false;
103 : }
104 1002 : if (OGRArrowWriterLayer::ICreateFeature(&oFeat) != OGRERR_NONE)
105 : {
106 0 : return false;
107 : }
108 :
109 1002 : if ((m_nFeatureCount % PROGRESS_FC_INTERVAL) == 0)
110 : {
111 0 : CPLDebugProgress(
112 : "PARQUET",
113 : "CopyTmpGpkgLayerToFinalFile(): %.02f%% progress",
114 0 : 100.0 * double(m_nFeatureCount) /
115 0 : double(m_nTmpFeatureCount));
116 : }
117 : }
118 :
119 3 : if (!FlushFeatures())
120 : {
121 0 : return false;
122 : }
123 : }
124 :
125 : // Now walk through the GPKG RTree for features with geometries
126 : // Cf https://github.com/sqlite/sqlite/blob/master/ext/rtree/rtree.c
127 : // for the description of the content of the rtree _node table
128 6 : std::vector<std::pair<int64_t, int>> aNodeNoDepthPair;
129 3 : int nTreeDepth = 0;
130 : // Queue the root node
131 : aNodeNoDepthPair.emplace_back(
132 3 : std::make_pair(/* nodeNo = */ 1, /* depth = */ 0));
133 3 : int nCountWrittenFeaturesSinceLastFlush = 0;
134 50 : while (!aNodeNoDepthPair.empty())
135 : {
136 47 : const auto &oLastPair = aNodeNoDepthPair.back();
137 47 : const int64_t nNodeNo = oLastPair.first;
138 47 : const int nCurDepth = oLastPair.second;
139 : //CPLDebug("PARQUET", "Reading nodeNode=%d, curDepth=%d", int(nNodeNo), nCurDepth);
140 47 : aNodeNoDepthPair.pop_back();
141 :
142 47 : auto poRTreeLayer = std::unique_ptr<OGRLayer>(m_poTmpGPKG->ExecuteSQL(
143 : CPLSPrintf("SELECT data FROM rtree_tmp_geom_node WHERE nodeno "
144 : "= " CPL_FRMT_GIB,
145 : static_cast<GIntBig>(nNodeNo)),
146 47 : nullptr, nullptr));
147 47 : if (!poRTreeLayer)
148 : {
149 0 : CPLError(CE_Failure, CPLE_AppDefined,
150 : "Cannot read node " CPL_FRMT_GIB,
151 : static_cast<GIntBig>(nNodeNo));
152 0 : return false;
153 : }
154 : const auto poRTreeFeature =
155 47 : std::unique_ptr<const OGRFeature>(poRTreeLayer->GetNextFeature());
156 47 : if (!poRTreeFeature)
157 : {
158 0 : CPLError(CE_Failure, CPLE_AppDefined,
159 : "Cannot read node " CPL_FRMT_GIB,
160 : static_cast<GIntBig>(nNodeNo));
161 0 : return false;
162 : }
163 :
164 47 : int nNodeBytes = 0;
165 : const GByte *pabyNodeData =
166 47 : poRTreeFeature->GetFieldAsBinary(0, &nNodeBytes);
167 47 : constexpr int BLOB_HEADER_SIZE = 4;
168 47 : if (nNodeBytes < BLOB_HEADER_SIZE)
169 : {
170 0 : CPLError(CE_Failure, CPLE_AppDefined,
171 : "Not enough bytes when reading node " CPL_FRMT_GIB,
172 : static_cast<GIntBig>(nNodeNo));
173 0 : return false;
174 : }
175 47 : if (nNodeNo == 1)
176 : {
177 : // Get the RTree depth from the root node
178 3 : nTreeDepth = (pabyNodeData[0] << 8) | pabyNodeData[1];
179 : //CPLDebug("PARQUET", "nTreeDepth = %d", nTreeDepth);
180 : }
181 :
182 47 : const int nCellCount = (pabyNodeData[2] << 8) | pabyNodeData[3];
183 47 : constexpr int SIZEOF_CELL = 24; // int64_t + 4 float
184 47 : if (nNodeBytes < BLOB_HEADER_SIZE + SIZEOF_CELL * nCellCount)
185 : {
186 0 : CPLError(CE_Failure, CPLE_AppDefined,
187 : "Not enough bytes when reading node " CPL_FRMT_GIB,
188 : static_cast<GIntBig>(nNodeNo));
189 0 : return false;
190 : }
191 :
192 47 : size_t nOffset = BLOB_HEADER_SIZE;
193 47 : if (nCurDepth == nTreeDepth)
194 : {
195 : // Leaf node: it references feature IDs.
196 :
197 : // If we are about to go above m_nRowGroupSize, flush past
198 : // features now, to improve the spatial compacity of the row group.
199 45 : if (m_nRowGroupSize > nCellCount &&
200 45 : nCountWrittenFeaturesSinceLastFlush + nCellCount >
201 45 : m_nRowGroupSize)
202 : {
203 14 : nCountWrittenFeaturesSinceLastFlush = 0;
204 14 : if (!FlushFeatures())
205 : {
206 0 : return false;
207 : }
208 : }
209 :
210 : // nCellCount shouldn't be over 51 normally, but even 65535
211 : // would be fine...
212 45 : assert(nCellCount <= 65535);
213 1247 : for (int i = 0; i < nCellCount; ++i)
214 : {
215 : int64_t nFID;
216 1202 : memcpy(&nFID, pabyNodeData + nOffset, sizeof(int64_t));
217 1202 : CPL_MSBPTR64(&nFID);
218 :
219 : const auto poSrcFeature = std::unique_ptr<const OGRFeature>(
220 1202 : m_poTmpGPKGLayer->GetFeature(nFID));
221 1202 : if (!poSrcFeature)
222 : {
223 0 : CPLError(CE_Failure, CPLE_AppDefined,
224 : "Cannot get feature " CPL_FRMT_GIB,
225 : static_cast<GIntBig>(nFID));
226 0 : return false;
227 : }
228 :
229 1202 : int nBytesFeature = 0;
230 : const GByte *pabyFeatureData =
231 1202 : poSrcFeature->GetFieldAsBinary(0, &nBytesFeature);
232 1202 : if (!oFeat.DeserializeFromBinary(pabyFeatureData,
233 : nBytesFeature))
234 : {
235 0 : CPLError(CE_Failure, CPLE_AppDefined,
236 : "Cannot deserialize feature");
237 0 : return false;
238 : }
239 1202 : if (OGRArrowWriterLayer::ICreateFeature(&oFeat) != OGRERR_NONE)
240 : {
241 0 : return false;
242 : }
243 :
244 1202 : nOffset += SIZEOF_CELL;
245 :
246 1202 : ++nCountWrittenFeaturesSinceLastFlush;
247 :
248 1202 : if ((m_nFeatureCount % PROGRESS_FC_INTERVAL) == 0 ||
249 1202 : m_nFeatureCount == m_nTmpFeatureCount / 2)
250 : {
251 2 : CPLDebugProgress(
252 : "PARQUET",
253 : "CopyTmpGpkgLayerToFinalFile(): %.02f%% progress",
254 2 : 100.0 * double(m_nFeatureCount) /
255 2 : double(m_nTmpFeatureCount));
256 : }
257 : }
258 : }
259 : else
260 : {
261 : // Non-leaf node: it references child nodes.
262 :
263 : // nCellCount shouldn't be over 51 normally, but even 65535
264 : // would be fine...
265 2 : assert(nCellCount <= 65535);
266 46 : for (int i = 0; i < nCellCount; ++i)
267 : {
268 : int64_t nNode;
269 44 : memcpy(&nNode, pabyNodeData + nOffset, sizeof(int64_t));
270 44 : CPL_MSBPTR64(&nNode);
271 : aNodeNoDepthPair.emplace_back(
272 44 : std::make_pair(nNode, nCurDepth + 1));
273 44 : nOffset += SIZEOF_CELL;
274 : }
275 : }
276 : }
277 :
278 3 : CPLDebug("PARQUET",
279 : "CopyTmpGpkgLayerToFinalFile(): 100%%, successfully finished");
280 3 : return true;
281 : }
282 :
283 : /************************************************************************/
284 : /* IsSupportedGeometryType() */
285 : /************************************************************************/
286 :
287 289 : bool OGRParquetWriterLayer::IsSupportedGeometryType(
288 : OGRwkbGeometryType eGType) const
289 : {
290 289 : const auto eFlattenType = wkbFlatten(eGType);
291 289 : if (!OGR_GT_HasM(eGType) && eFlattenType <= wkbGeometryCollection)
292 : {
293 288 : return true;
294 : }
295 :
296 : const auto osConfigOptionName =
297 3 : "OGR_" + GetDriverUCName() + "_ALLOW_ALL_DIMS";
298 1 : if (CPLTestBool(CPLGetConfigOption(osConfigOptionName.c_str(), "NO")))
299 : {
300 0 : return true;
301 : }
302 :
303 1 : CPLError(CE_Failure, CPLE_NotSupported,
304 : "Only 2D and Z geometry types are supported (unless the "
305 : "%s configuration option is set to YES)",
306 : osConfigOptionName.c_str());
307 1 : return false;
308 : }
309 :
310 : /************************************************************************/
311 : /* SetOptions() */
312 : /************************************************************************/
313 :
314 293 : bool OGRParquetWriterLayer::SetOptions(
315 : const OGRGeomFieldDefn *poSrcGeomFieldDefn, CSLConstList papszOptions)
316 : {
317 293 : m_aosCreationOptions = papszOptions;
318 :
319 293 : const char *pszWriteCoveringBBox = CSLFetchNameValueDef(
320 : papszOptions, "WRITE_COVERING_BBOX",
321 : CPLGetConfigOption("OGR_PARQUET_WRITE_COVERING_BBOX", nullptr));
322 293 : m_bWriteBBoxStruct =
323 293 : pszWriteCoveringBBox == nullptr || CPLTestBool(pszWriteCoveringBBox);
324 :
325 293 : if (CPLTestBool(CSLFetchNameValueDef(papszOptions, "SORT_BY_BBOX", "NO")))
326 : {
327 8 : const std::string osTmpGPKG(std::string(m_poDataset->GetDescription()) +
328 4 : ".tmp.gpkg");
329 4 : auto poGPKGDrv = GetGDALDriverManager()->GetDriverByName("GPKG");
330 4 : if (!poGPKGDrv)
331 : {
332 1 : CPLError(
333 : CE_Failure, CPLE_AppDefined,
334 : "Driver GPKG required for SORT_BY_BBOX layer creation option");
335 1 : return false;
336 : }
337 3 : m_poTmpGPKG.reset(poGPKGDrv->Create(osTmpGPKG.c_str(), 0, 0, 0,
338 : GDT_Unknown, nullptr));
339 3 : if (!m_poTmpGPKG)
340 0 : return false;
341 3 : m_poTmpGPKG->MarkSuppressOnClose();
342 3 : m_poTmpGPKGLayer = m_poTmpGPKG->CreateLayer("tmp");
343 12 : if (!m_poTmpGPKGLayer ||
344 : // Serialized feature
345 3 : m_poTmpGPKGLayer->CreateField(
346 3 : std::make_unique<OGRFieldDefn>("serialized_feature", OFTBinary)
347 3 : .get()) != OGRERR_NONE ||
348 : // FlushCache is needed to avoid SQLite3 errors on empty layers
349 9 : m_poTmpGPKG->FlushCache() != CE_None ||
350 3 : m_poTmpGPKGLayer->StartTransaction() != OGRERR_NONE)
351 : {
352 0 : return false;
353 : }
354 : }
355 :
356 : const char *pszGeomEncoding =
357 292 : CSLFetchNameValue(papszOptions, "GEOMETRY_ENCODING");
358 292 : m_eGeomEncoding = OGRArrowGeomEncoding::WKB;
359 292 : if (pszGeomEncoding)
360 : {
361 148 : if (EQUAL(pszGeomEncoding, "WKB"))
362 0 : m_eGeomEncoding = OGRArrowGeomEncoding::WKB;
363 148 : else if (EQUAL(pszGeomEncoding, "WKT"))
364 8 : m_eGeomEncoding = OGRArrowGeomEncoding::WKT;
365 140 : else if (EQUAL(pszGeomEncoding, "GEOARROW_INTERLEAVED"))
366 : {
367 28 : CPLErrorOnce(
368 : CE_Warning, CPLE_AppDefined,
369 : "Use of GEOMETRY_ENCODING=GEOARROW_INTERLEAVED is not "
370 : "recommended. "
371 : "GeoParquet 1.1 uses GEOMETRY_ENCODING=GEOARROW (struct) "
372 : "instead.");
373 28 : m_eGeomEncoding = OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC;
374 : }
375 112 : else if (EQUAL(pszGeomEncoding, "GEOARROW") ||
376 0 : EQUAL(pszGeomEncoding, "GEOARROW_STRUCT"))
377 112 : m_eGeomEncoding = OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC;
378 : else
379 : {
380 0 : CPLError(CE_Failure, CPLE_NotSupported,
381 : "Unsupported GEOMETRY_ENCODING = %s", pszGeomEncoding);
382 0 : return false;
383 : }
384 : }
385 :
386 : const char *pszCoordPrecision =
387 292 : CSLFetchNameValue(papszOptions, "COORDINATE_PRECISION");
388 292 : if (pszCoordPrecision)
389 0 : m_nWKTCoordinatePrecision = atoi(pszCoordPrecision);
390 :
391 292 : m_bForceCounterClockwiseOrientation =
392 292 : EQUAL(CSLFetchNameValueDef(papszOptions, "POLYGON_ORIENTATION",
393 : "COUNTERCLOCKWISE"),
394 : "COUNTERCLOCKWISE");
395 :
396 : const auto eGType =
397 292 : poSrcGeomFieldDefn ? poSrcGeomFieldDefn->GetType() : wkbNone;
398 292 : if (poSrcGeomFieldDefn && eGType != wkbNone)
399 : {
400 263 : if (!IsSupportedGeometryType(eGType))
401 : {
402 1 : return false;
403 : }
404 :
405 262 : m_poFeatureDefn->SetGeomType(eGType);
406 262 : auto eGeomEncoding = m_eGeomEncoding;
407 262 : if (eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC ||
408 234 : eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC)
409 : {
410 140 : const auto eEncodingType = eGeomEncoding;
411 140 : eGeomEncoding = GetPreciseArrowGeomEncoding(eEncodingType, eGType);
412 140 : if (eGeomEncoding == eEncodingType)
413 0 : return false;
414 : }
415 262 : m_aeGeomEncoding.push_back(eGeomEncoding);
416 :
417 524 : std::string osGeometryName;
418 : const char *pszGeometryName =
419 262 : CSLFetchNameValue(papszOptions, "GEOMETRY_NAME");
420 262 : if (pszGeometryName)
421 9 : osGeometryName = pszGeometryName;
422 253 : else if (poSrcGeomFieldDefn->GetNameRef()[0])
423 4 : osGeometryName = poSrcGeomFieldDefn->GetNameRef();
424 : else
425 249 : osGeometryName = "geometry";
426 262 : m_poFeatureDefn->GetGeomFieldDefn(0)->SetName(osGeometryName.c_str());
427 :
428 262 : const auto poSpatialRef = poSrcGeomFieldDefn->GetSpatialRef();
429 262 : if (poSpatialRef)
430 : {
431 35 : auto poSRS = poSpatialRef->Clone();
432 35 : m_poFeatureDefn->GetGeomFieldDefn(0)->SetSpatialRef(poSRS);
433 35 : poSRS->Release();
434 : }
435 : }
436 :
437 291 : m_osFIDColumn = CSLFetchNameValueDef(papszOptions, "FID", "");
438 :
439 291 : const char *pszCompression = CSLFetchNameValue(papszOptions, "COMPRESSION");
440 291 : if (pszCompression == nullptr)
441 : {
442 855 : auto oResult = arrow::util::Codec::GetCompressionType("snappy");
443 285 : if (oResult.ok() && arrow::util::Codec::IsAvailable(*oResult))
444 : {
445 285 : pszCompression = "SNAPPY";
446 : }
447 : else
448 : {
449 0 : pszCompression = "NONE";
450 : }
451 : }
452 :
453 291 : if (EQUAL(pszCompression, "NONE"))
454 0 : pszCompression = "UNCOMPRESSED";
455 : auto oResult = arrow::util::Codec::GetCompressionType(
456 582 : CPLString(pszCompression).tolower());
457 291 : if (!oResult.ok())
458 : {
459 1 : CPLError(CE_Failure, CPLE_NotSupported,
460 : "Unrecognized compression method: %s", pszCompression);
461 1 : return false;
462 : }
463 290 : m_eCompression = *oResult;
464 290 : if (!arrow::util::Codec::IsAvailable(m_eCompression))
465 : {
466 0 : CPLError(CE_Failure, CPLE_NotSupported,
467 : "Compression method %s is known, but libarrow has not "
468 : "been built with support for it",
469 : pszCompression);
470 0 : return false;
471 : }
472 290 : m_oWriterPropertiesBuilder.compression(m_eCompression);
473 :
474 : const char *pszCompressionLevel =
475 290 : CSLFetchNameValue(papszOptions, "COMPRESSION_LEVEL");
476 290 : if (pszCompressionLevel)
477 : {
478 2 : const int nCompressionLevel = atoi(pszCompressionLevel);
479 2 : if (nCompressionLevel != DEFAULT_COMPRESSION_LEVEL)
480 2 : m_oWriterPropertiesBuilder.compression_level(nCompressionLevel);
481 : }
482 288 : else if (EQUAL(pszCompression, "ZSTD"))
483 1 : m_oWriterPropertiesBuilder.compression_level(
484 : OGR_PARQUET_ZSTD_DEFAULT_COMPRESSION_LEVEL);
485 :
486 : const std::string osCreator =
487 290 : CSLFetchNameValueDef(papszOptions, "CREATOR", "");
488 290 : if (!osCreator.empty())
489 1 : m_oWriterPropertiesBuilder.created_by(osCreator);
490 : else
491 289 : m_oWriterPropertiesBuilder.created_by("GDAL " GDAL_RELEASE_NAME
492 : ", using " CREATED_BY_VERSION);
493 :
494 : // Undocumented option. Not clear it is useful besides unit test purposes
495 290 : if (!CPLTestBool(CSLFetchNameValueDef(papszOptions, "STATISTICS", "YES")))
496 1 : m_oWriterPropertiesBuilder.disable_statistics();
497 :
498 : #if PARQUET_VERSION_MAJOR >= 12
499 : // Undocumented option. Not clear it is useful to disable it.
500 290 : if (CPLTestBool(CSLFetchNameValueDef(papszOptions, "PAGE_INDEX", "YES")))
501 290 : m_oWriterPropertiesBuilder.enable_write_page_index();
502 : #endif
503 :
504 : const char *pszWriteGeo =
505 290 : CPLGetConfigOption("OGR_PARQUET_WRITE_GEO", nullptr);
506 290 : m_bWriteGeoMetadata = pszWriteGeo == nullptr || CPLTestBool(pszWriteGeo);
507 :
508 290 : if (m_eGeomEncoding == OGRArrowGeomEncoding::WKB && eGType != wkbNone)
509 : {
510 : #if ARROW_VERSION_MAJOR >= 21
511 : const char *pszUseParquetGeoTypes =
512 : CSLFetchNameValueDef(papszOptions, "USE_PARQUET_GEO_TYPES", "NO");
513 : if (EQUAL(pszUseParquetGeoTypes, "ONLY"))
514 : {
515 : m_bUseArrowWKBExtension = true;
516 : if (pszWriteGeo == nullptr)
517 : m_bWriteGeoMetadata = false;
518 : if (pszWriteCoveringBBox == nullptr)
519 : m_bWriteBBoxStruct = false;
520 : }
521 : else
522 : {
523 : m_bUseArrowWKBExtension = CPLTestBool(pszUseParquetGeoTypes);
524 : }
525 : #else
526 121 : m_oWriterPropertiesBuilder.disable_statistics(
527 363 : parquet::schema::ColumnPath::FromDotString(
528 121 : m_poFeatureDefn->GetGeomFieldDefn(0)->GetNameRef()));
529 : #endif
530 : }
531 :
532 : const char *pszRowGroupSize =
533 290 : CSLFetchNameValue(papszOptions, "ROW_GROUP_SIZE");
534 290 : if (pszRowGroupSize)
535 : {
536 12 : auto nRowGroupSize = static_cast<int64_t>(atoll(pszRowGroupSize));
537 12 : if (nRowGroupSize > 0)
538 : {
539 12 : if (nRowGroupSize > INT_MAX)
540 0 : nRowGroupSize = INT_MAX;
541 12 : m_nRowGroupSize = nRowGroupSize;
542 : }
543 : }
544 :
545 290 : m_bEdgesSpherical = EQUAL(
546 : CSLFetchNameValueDef(papszOptions, "EDGES", "PLANAR"), "SPHERICAL");
547 :
548 290 : m_bInitializationOK = true;
549 290 : return true;
550 : }
551 :
552 : /************************************************************************/
553 : /* CloseFileWriter() */
554 : /************************************************************************/
555 :
556 290 : bool OGRParquetWriterLayer::CloseFileWriter()
557 : {
558 580 : auto status = m_poFileWriter->Close();
559 290 : if (!status.ok())
560 : {
561 0 : CPLError(CE_Failure, CPLE_AppDefined,
562 : "FileWriter::Close() failed with %s",
563 0 : status.message().c_str());
564 : }
565 580 : return status.ok();
566 : }
567 :
568 : /************************************************************************/
569 : /* GetGeoMetadata() */
570 : /************************************************************************/
571 :
572 290 : std::string OGRParquetWriterLayer::GetGeoMetadata() const
573 : {
574 : // Just for unit testing purposes
575 : const char *pszGeoMetadata =
576 290 : CPLGetConfigOption("OGR_PARQUET_GEO_METADATA", nullptr);
577 290 : if (pszGeoMetadata)
578 16 : return pszGeoMetadata;
579 :
580 274 : if (m_poFeatureDefn->GetGeomFieldCount() != 0 && m_bWriteGeoMetadata)
581 : {
582 508 : CPLJSONObject oRoot;
583 254 : oRoot.Add("version", "1.1.0");
584 254 : oRoot.Add("primary_column",
585 254 : m_poFeatureDefn->GetGeomFieldDefn(0)->GetNameRef());
586 508 : CPLJSONObject oColumns;
587 254 : oRoot.Add("columns", oColumns);
588 525 : for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
589 : {
590 271 : const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
591 542 : CPLJSONObject oColumn;
592 271 : oColumns.Add(poGeomFieldDefn->GetNameRef(), oColumn);
593 271 : oColumn.Add("encoding",
594 271 : GetGeomEncodingAsString(m_aeGeomEncoding[i], true));
595 :
596 271 : if (CPLTestBool(CPLGetConfigOption("OGR_PARQUET_WRITE_CRS", "YES")))
597 : {
598 270 : const auto poSRS = poGeomFieldDefn->GetSpatialRef();
599 270 : if (poSRS)
600 : {
601 68 : OGRSpatialReference oSRSIdentified(IdentifyCRS(poSRS));
602 :
603 : const char *pszAuthName =
604 34 : oSRSIdentified.GetAuthorityName(nullptr);
605 : const char *pszAuthCode =
606 34 : oSRSIdentified.GetAuthorityCode(nullptr);
607 :
608 34 : bool bOmitCRS = false;
609 34 : if (pszAuthName != nullptr && pszAuthCode != nullptr &&
610 33 : ((EQUAL(pszAuthName, "EPSG") &&
611 30 : EQUAL(pszAuthCode, "4326")) ||
612 22 : (EQUAL(pszAuthName, "OGC") &&
613 3 : EQUAL(pszAuthCode, "CRS84"))))
614 : {
615 : // To make things less confusing for non-geo-aware
616 : // consumers, omit EPSG:4326 / OGC:CRS84 CRS by default
617 14 : bOmitCRS = CPLTestBool(CPLGetConfigOption(
618 : "OGR_PARQUET_CRS_OMIT_IF_WGS84", "YES"));
619 : }
620 :
621 34 : if (bOmitCRS)
622 : {
623 : // do nothing
624 : }
625 20 : else if (EQUAL(CPLGetConfigOption(
626 : "OGR_PARQUET_CRS_ENCODING", "PROJJSON"),
627 : "PROJJSON"))
628 : {
629 : // CRS encoded as PROJJSON for GeoParquet >= 0.4.0
630 20 : char *pszPROJJSON = nullptr;
631 20 : oSRSIdentified.exportToPROJJSON(&pszPROJJSON, nullptr);
632 40 : CPLJSONDocument oCRSDoc;
633 20 : CPL_IGNORE_RET_VAL(oCRSDoc.LoadMemory(pszPROJJSON));
634 20 : CPLFree(pszPROJJSON);
635 20 : CPLJSONObject oCRSRoot = oCRSDoc.GetRoot();
636 20 : RemoveIDFromMemberOfEnsembles(oCRSRoot);
637 20 : oColumn.Add("crs", oCRSRoot);
638 : }
639 : else
640 : {
641 : // WKT was used in GeoParquet <= 0.3.0
642 0 : const char *const apszOptions[] = {
643 : "FORMAT=WKT2_2019", "MULTILINE=NO", nullptr};
644 0 : char *pszWKT = nullptr;
645 0 : oSRSIdentified.exportToWkt(&pszWKT, apszOptions);
646 0 : if (pszWKT)
647 0 : oColumn.Add("crs", pszWKT);
648 0 : CPLFree(pszWKT);
649 : }
650 :
651 34 : const double dfCoordEpoch = poSRS->GetCoordinateEpoch();
652 34 : if (dfCoordEpoch > 0)
653 2 : oColumn.Add("epoch", dfCoordEpoch);
654 : }
655 : else
656 : {
657 236 : oColumn.AddNull("crs");
658 : }
659 : }
660 :
661 271 : if (m_bEdgesSpherical)
662 : {
663 3 : oColumn.Add("edges", "spherical");
664 : }
665 :
666 509 : if (m_aoEnvelopes[i].IsInit() &&
667 238 : CPLTestBool(
668 : CPLGetConfigOption("OGR_PARQUET_WRITE_BBOX", "YES")))
669 : {
670 238 : bool bHasZ = false;
671 435 : for (const auto eGeomType : m_oSetWrittenGeometryTypes[i])
672 : {
673 280 : bHasZ = OGR_GT_HasZ(eGeomType);
674 280 : if (bHasZ)
675 83 : break;
676 : }
677 238 : CPLJSONArray oBBOX;
678 238 : oBBOX.Add(m_aoEnvelopes[i].MinX);
679 238 : oBBOX.Add(m_aoEnvelopes[i].MinY);
680 238 : if (bHasZ)
681 83 : oBBOX.Add(m_aoEnvelopes[i].MinZ);
682 238 : oBBOX.Add(m_aoEnvelopes[i].MaxX);
683 238 : oBBOX.Add(m_aoEnvelopes[i].MaxY);
684 238 : if (bHasZ)
685 83 : oBBOX.Add(m_aoEnvelopes[i].MaxZ);
686 238 : oColumn.Add("bbox", oBBOX);
687 : }
688 :
689 : // Bounding box column definition
690 472 : if (m_bWriteBBoxStruct &&
691 201 : CPLTestBool(CPLGetConfigOption(
692 : "OGR_PARQUET_WRITE_COVERING_BBOX_IN_METADATA", "YES")))
693 : {
694 402 : CPLJSONObject oCovering;
695 201 : oColumn.Add("covering", oCovering);
696 402 : CPLJSONObject oBBOX;
697 201 : oCovering.Add("bbox", oBBOX);
698 : const auto AddComponent =
699 2412 : [this, i, &oBBOX](const char *pszComponent)
700 : {
701 804 : CPLJSONArray oArray;
702 804 : oArray.Add(m_apoFieldsBBOX[i]->name());
703 804 : oArray.Add(pszComponent);
704 804 : oBBOX.Add(pszComponent, oArray);
705 804 : };
706 201 : AddComponent("xmin");
707 201 : AddComponent("ymin");
708 201 : AddComponent("xmax");
709 201 : AddComponent("ymax");
710 : }
711 :
712 296 : const auto GetStringGeometryType = [](OGRwkbGeometryType eType)
713 : {
714 296 : const auto eFlattenType = wkbFlatten(eType);
715 296 : std::string osType = "Unknown";
716 296 : if (wkbPoint == eFlattenType)
717 77 : osType = "Point";
718 219 : else if (wkbLineString == eFlattenType)
719 34 : osType = "LineString";
720 185 : else if (wkbPolygon == eFlattenType)
721 56 : osType = "Polygon";
722 129 : else if (wkbMultiPoint == eFlattenType)
723 26 : osType = "MultiPoint";
724 103 : else if (wkbMultiLineString == eFlattenType)
725 29 : osType = "MultiLineString";
726 74 : else if (wkbMultiPolygon == eFlattenType)
727 69 : osType = "MultiPolygon";
728 5 : else if (wkbGeometryCollection == eFlattenType)
729 5 : osType = "GeometryCollection";
730 296 : if (osType != "Unknown")
731 : {
732 : // M and ZM not supported officially currently, but it
733 : // doesn't hurt to anticipate
734 296 : if (OGR_GT_HasZ(eType) && OGR_GT_HasM(eType))
735 8 : osType += " ZM";
736 288 : else if (OGR_GT_HasZ(eType))
737 91 : osType += " Z";
738 197 : else if (OGR_GT_HasM(eType))
739 8 : osType += " M";
740 : }
741 296 : return osType;
742 : };
743 :
744 271 : if (m_bForceCounterClockwiseOrientation)
745 270 : oColumn.Add("orientation", "counterclockwise");
746 :
747 271 : CPLJSONArray oArray;
748 567 : for (const auto eType : m_oSetWrittenGeometryTypes[i])
749 : {
750 296 : oArray.Add(GetStringGeometryType(eType));
751 : }
752 271 : oColumn.Add("geometry_types", oArray);
753 : }
754 :
755 254 : return oRoot.Format(CPLJSONObject::PrettyFormat::Plain);
756 : }
757 20 : return std::string();
758 : }
759 :
760 : /************************************************************************/
761 : /* PerformStepsBeforeFinalFlushGroup() */
762 : /************************************************************************/
763 :
764 290 : void OGRParquetWriterLayer::PerformStepsBeforeFinalFlushGroup()
765 : {
766 290 : if (m_poKeyValueMetadata)
767 : {
768 580 : std::string osGeoMetadata = GetGeoMetadata();
769 580 : auto poTmpSchema = m_poSchema;
770 290 : if (!osGeoMetadata.empty())
771 : {
772 : // HACK: it would be good for Arrow to provide a clean way to alter
773 : // key value metadata before finalizing.
774 : // We need to write metadata at end to write the bounding box.
775 270 : const_cast<arrow::KeyValueMetadata *>(m_poKeyValueMetadata.get())
776 270 : ->Append("geo", osGeoMetadata);
777 :
778 270 : auto kvMetadata = poTmpSchema->metadata()
779 15 : ? poTmpSchema->metadata()->Copy()
780 285 : : std::make_shared<arrow::KeyValueMetadata>();
781 270 : kvMetadata->Append("geo", std::move(osGeoMetadata));
782 270 : poTmpSchema = poTmpSchema->WithMetadata(kvMetadata);
783 : }
784 :
785 290 : if (CPLTestBool(
786 : CPLGetConfigOption("OGR_PARQUET_WRITE_ARROW_SCHEMA", "YES")))
787 : {
788 : auto status =
789 580 : ::arrow::ipc::SerializeSchema(*poTmpSchema, m_poMemoryPool);
790 290 : if (status.ok())
791 : {
792 : // The serialized schema is not UTF-8, which is required for
793 : // Thrift
794 580 : const std::string schema_as_string = (*status)->ToString();
795 : std::string schema_base64 =
796 290 : ::arrow::util::base64_encode(schema_as_string);
797 290 : static const std::string kArrowSchemaKey = "ARROW:schema";
798 : const_cast<arrow::KeyValueMetadata *>(
799 290 : m_poKeyValueMetadata.get())
800 290 : ->Append(kArrowSchemaKey, std::move(schema_base64));
801 : }
802 : }
803 :
804 : // Put GDAL metadata into a gdal:metadata domain
805 580 : CPLJSONObject oMultiMetadata;
806 290 : bool bHasMultiMetadata = false;
807 297 : auto &l_oMDMD = oMDMD.GetDomainList() && *(oMDMD.GetDomainList())
808 297 : ? oMDMD
809 283 : : m_poDataset->GetMultiDomainMetadata();
810 299 : for (CSLConstList papszDomainIter = l_oMDMD.GetDomainList();
811 299 : papszDomainIter && *papszDomainIter; ++papszDomainIter)
812 : {
813 9 : const char *pszDomain = *papszDomainIter;
814 9 : CSLConstList papszMD = l_oMDMD.GetMetadata(pszDomain);
815 9 : if (STARTS_WITH(pszDomain, "json:") && papszMD && papszMD[0])
816 : {
817 1 : CPLJSONDocument oDoc;
818 1 : if (oDoc.LoadMemory(papszMD[0]))
819 : {
820 1 : bHasMultiMetadata = true;
821 1 : oMultiMetadata.Add(pszDomain, oDoc.GetRoot());
822 1 : continue;
823 0 : }
824 : }
825 8 : else if (STARTS_WITH(pszDomain, "xml:") && papszMD && papszMD[0])
826 : {
827 1 : bHasMultiMetadata = true;
828 1 : oMultiMetadata.Add(pszDomain, papszMD[0]);
829 1 : continue;
830 : }
831 14 : CPLJSONObject oMetadata;
832 7 : bool bHasMetadata = false;
833 14 : for (CSLConstList papszMDIter = papszMD;
834 14 : papszMDIter && *papszMDIter; ++papszMDIter)
835 : {
836 7 : char *pszKey = nullptr;
837 7 : const char *pszValue = CPLParseNameValue(*papszMDIter, &pszKey);
838 7 : if (pszKey && pszValue)
839 : {
840 7 : bHasMetadata = true;
841 7 : bHasMultiMetadata = true;
842 7 : oMetadata.Add(pszKey, pszValue);
843 : }
844 7 : CPLFree(pszKey);
845 : }
846 7 : if (bHasMetadata)
847 7 : oMultiMetadata.Add(pszDomain, oMetadata);
848 : }
849 290 : if (bHasMultiMetadata)
850 : {
851 7 : const_cast<arrow::KeyValueMetadata *>(m_poKeyValueMetadata.get())
852 7 : ->Append(
853 : "gdal:metadata",
854 14 : oMultiMetadata.Format(CPLJSONObject::PrettyFormat::Plain));
855 : }
856 :
857 290 : if (!m_aosCreationOptions.empty())
858 : {
859 370 : CPLJSONObject oCreationOptions;
860 185 : bool bEmpty = true;
861 688 : for (const auto &[key, value] :
862 873 : cpl::IterateNameValue(m_aosCreationOptions))
863 : {
864 344 : if (!EQUAL(key, "FID") && !EQUAL(key, "GEOMETRY_NAME") &&
865 319 : !EQUAL(key, "EDGES"))
866 : {
867 315 : bEmpty = false;
868 315 : oCreationOptions.Add(key, value);
869 : }
870 : }
871 185 : if (!bEmpty)
872 : {
873 : const_cast<arrow::KeyValueMetadata *>(
874 171 : m_poKeyValueMetadata.get())
875 171 : ->Append("gdal:creation-options",
876 342 : oCreationOptions.Format(
877 : CPLJSONObject::PrettyFormat::Plain));
878 : }
879 : }
880 : }
881 290 : }
882 :
883 : /************************************************************************/
884 : /* Open() */
885 : /************************************************************************/
886 :
887 : // Same as parquet::arrow::FileWriter::Open(), except we also
888 : // return KeyValueMetadata
889 : static arrow::Status
890 290 : Open(const ::arrow::Schema &schema, ::arrow::MemoryPool *pool,
891 : std::shared_ptr<::arrow::io::OutputStream> sink,
892 : std::shared_ptr<parquet::WriterProperties> properties,
893 : std::shared_ptr<parquet::ArrowWriterProperties> arrow_properties,
894 : std::unique_ptr<parquet::arrow::FileWriter> *writer,
895 : std::shared_ptr<const arrow::KeyValueMetadata> *outMetadata)
896 : {
897 290 : std::shared_ptr<parquet::SchemaDescriptor> parquet_schema;
898 580 : RETURN_NOT_OK(parquet::arrow::ToParquetSchema(
899 : &schema, *properties, *arrow_properties, &parquet_schema));
900 :
901 : auto schema_node = std::static_pointer_cast<parquet::schema::GroupNode>(
902 580 : parquet_schema->schema_root());
903 :
904 290 : auto metadata = schema.metadata()
905 20 : ? schema.metadata()->Copy()
906 600 : : std::make_shared<arrow::KeyValueMetadata>();
907 290 : *outMetadata = metadata;
908 :
909 290 : std::unique_ptr<parquet::ParquetFileWriter> base_writer;
910 290 : PARQUET_CATCH_NOT_OK(base_writer = parquet::ParquetFileWriter::Open(
911 : std::move(sink), std::move(schema_node),
912 : std::move(properties), metadata));
913 :
914 290 : auto schema_ptr = std::make_shared<::arrow::Schema>(schema);
915 : return parquet::arrow::FileWriter::Make(
916 580 : pool, std::move(base_writer), std::move(schema_ptr),
917 870 : std::move(arrow_properties), writer);
918 : }
919 :
920 : /************************************************************************/
921 : /* CreateSchema() */
922 : /************************************************************************/
923 :
924 290 : void OGRParquetWriterLayer::CreateSchema()
925 : {
926 290 : CreateSchemaCommon();
927 290 : }
928 :
929 : /************************************************************************/
930 : /* CreateGeomField() */
931 : /************************************************************************/
932 :
933 27 : OGRErr OGRParquetWriterLayer::CreateGeomField(const OGRGeomFieldDefn *poField,
934 : int bApproxOK)
935 : {
936 27 : OGRErr eErr = OGRArrowWriterLayer::CreateGeomField(poField, bApproxOK);
937 53 : if (eErr == OGRERR_NONE &&
938 26 : m_aeGeomEncoding.back() == OGRArrowGeomEncoding::WKB
939 : #if ARROW_VERSION_MAJOR < 21
940 : // Geostatistics in Arrow 21 do not support geographic type for now
941 53 : && m_bEdgesSpherical
942 : #endif
943 : )
944 : {
945 0 : m_oWriterPropertiesBuilder.disable_statistics(
946 0 : parquet::schema::ColumnPath::FromDotString(
947 0 : m_poFeatureDefn
948 0 : ->GetGeomFieldDefn(m_poFeatureDefn->GetGeomFieldCount() - 1)
949 : ->GetNameRef()));
950 : }
951 27 : return eErr;
952 : }
953 :
954 : /************************************************************************/
955 : /* CreateWriter() */
956 : /************************************************************************/
957 :
958 290 : void OGRParquetWriterLayer::CreateWriter()
959 : {
960 290 : CPLAssert(m_poFileWriter == nullptr);
961 :
962 290 : if (m_poSchema == nullptr)
963 : {
964 41 : CreateSchema();
965 : }
966 : else
967 : {
968 249 : FinalizeSchema();
969 : }
970 :
971 : auto arrowWriterProperties =
972 290 : parquet::ArrowWriterProperties::Builder().store_schema()->build();
973 870 : CPL_IGNORE_RET_VAL(Open(*m_poSchema, m_poMemoryPool, m_poOutputStream,
974 580 : m_oWriterPropertiesBuilder.build(),
975 290 : std::move(arrowWriterProperties), &m_poFileWriter,
976 : &m_poKeyValueMetadata));
977 290 : }
978 :
979 : /************************************************************************/
980 : /* ICreateFeature() */
981 : /************************************************************************/
982 :
983 3126 : OGRErr OGRParquetWriterLayer::ICreateFeature(OGRFeature *poFeature)
984 : {
985 : // If not using SORT_BY_BBOX=YES layer creation option, we can directly
986 : // write features to the final Parquet file
987 3126 : if (!m_poTmpGPKGLayer)
988 922 : return OGRArrowWriterLayer::ICreateFeature(poFeature);
989 :
990 : // SORT_BY_BBOX=YES case: we write for now a serialized version of poFeature
991 : // in a temporary GeoPackage file.
992 :
993 2204 : GIntBig nFID = poFeature->GetFID();
994 2204 : if (!m_osFIDColumn.empty() && nFID == OGRNullFID)
995 : {
996 1102 : nFID = m_nTmpFeatureCount;
997 1102 : poFeature->SetFID(nFID);
998 : }
999 2204 : ++m_nTmpFeatureCount;
1000 :
1001 4408 : std::vector<GByte> abyBuffer;
1002 : // Serialize the source feature as a single array of bytes to preserve it
1003 : // fully
1004 2204 : if (!poFeature->SerializeToBinary(abyBuffer))
1005 : {
1006 0 : return OGRERR_FAILURE;
1007 : }
1008 :
1009 : // SQLite3 limitation: a row must fit in slightly less than 1 GB.
1010 2204 : constexpr int SOME_MARGIN = 128;
1011 2204 : if (abyBuffer.size() > 1024 * 1024 * 1024 - SOME_MARGIN)
1012 : {
1013 0 : CPLError(CE_Failure, CPLE_NotSupported,
1014 : "Features larger than 1 GB are not supported");
1015 0 : return OGRERR_FAILURE;
1016 : }
1017 :
1018 4408 : OGRFeature oFeat(m_poTmpGPKGLayer->GetLayerDefn());
1019 2204 : oFeat.SetFID(nFID);
1020 2204 : oFeat.SetField(0, static_cast<int>(abyBuffer.size()), abyBuffer.data());
1021 2204 : const auto poSrcGeom = poFeature->GetGeometryRef();
1022 2204 : if (poSrcGeom && !poSrcGeom->IsEmpty())
1023 : {
1024 : // For the purpose of building an RTree, just use the bounding box of
1025 : // the geometry as the geometry.
1026 1202 : OGREnvelope sEnvelope;
1027 1202 : poSrcGeom->getEnvelope(&sEnvelope);
1028 2404 : auto poPoly = std::make_unique<OGRPolygon>();
1029 2404 : auto poLR = std::make_unique<OGRLinearRing>();
1030 1202 : poLR->addPoint(sEnvelope.MinX, sEnvelope.MinY);
1031 1202 : poLR->addPoint(sEnvelope.MinX, sEnvelope.MaxY);
1032 1202 : poLR->addPoint(sEnvelope.MaxX, sEnvelope.MaxY);
1033 1202 : poLR->addPoint(sEnvelope.MaxX, sEnvelope.MinY);
1034 1202 : poLR->addPoint(sEnvelope.MinX, sEnvelope.MinY);
1035 1202 : poPoly->addRingDirectly(poLR.release());
1036 1202 : oFeat.SetGeometryDirectly(poPoly.release());
1037 : }
1038 2204 : return m_poTmpGPKGLayer->CreateFeature(&oFeat);
1039 : }
1040 :
1041 : /************************************************************************/
1042 : /* FlushGroup() */
1043 : /************************************************************************/
1044 :
1045 274 : bool OGRParquetWriterLayer::FlushGroup()
1046 : {
1047 : #if PARQUET_VERSION_MAJOR >= 20
1048 : auto status = m_poFileWriter->NewRowGroup();
1049 : #else
1050 548 : auto status = m_poFileWriter->NewRowGroup(m_apoBuilders[0]->length());
1051 : #endif
1052 274 : if (!status.ok())
1053 : {
1054 0 : CPLError(CE_Failure, CPLE_AppDefined, "NewRowGroup() failed with %s",
1055 0 : status.message().c_str());
1056 0 : ClearArrayBuilers();
1057 0 : return false;
1058 : }
1059 :
1060 274 : auto ret = WriteArrays(
1061 1111 : [this](const std::shared_ptr<arrow::Field> &field,
1062 1111 : const std::shared_ptr<arrow::Array> &array)
1063 : {
1064 2222 : auto l_status = m_poFileWriter->WriteColumnChunk(*array);
1065 1111 : if (!l_status.ok())
1066 : {
1067 0 : CPLError(CE_Failure, CPLE_AppDefined,
1068 : "WriteColumnChunk() failed for field %s: %s",
1069 0 : field->name().c_str(), l_status.message().c_str());
1070 0 : return false;
1071 : }
1072 1111 : return true;
1073 : });
1074 :
1075 274 : ClearArrayBuilers();
1076 274 : return ret;
1077 : }
1078 :
1079 : /************************************************************************/
1080 : /* FixupWKBGeometryBeforeWriting() */
1081 : /************************************************************************/
1082 :
1083 43 : void OGRParquetWriterLayer::FixupWKBGeometryBeforeWriting(GByte *pabyWkb,
1084 : size_t nLen)
1085 : {
1086 43 : if (!m_bForceCounterClockwiseOrientation)
1087 0 : return;
1088 :
1089 43 : OGRWKBFixupCounterClockWiseExternalRing(pabyWkb, nLen);
1090 : }
1091 :
1092 : /************************************************************************/
1093 : /* FixupGeometryBeforeWriting() */
1094 : /************************************************************************/
1095 :
1096 1379 : void OGRParquetWriterLayer::FixupGeometryBeforeWriting(OGRGeometry *poGeom)
1097 : {
1098 1379 : if (!m_bForceCounterClockwiseOrientation)
1099 3 : return;
1100 :
1101 1376 : const auto eFlattenType = wkbFlatten(poGeom->getGeometryType());
1102 : // Polygon rings MUST follow the right-hand rule for orientation
1103 : // (counterclockwise external rings, clockwise internal rings)
1104 1376 : if (eFlattenType == wkbPolygon)
1105 : {
1106 74 : bool bFirstRing = true;
1107 151 : for (auto poRing : poGeom->toPolygon())
1108 : {
1109 85 : if ((bFirstRing && poRing->isClockwise()) ||
1110 8 : (!bFirstRing && !poRing->isClockwise()))
1111 : {
1112 72 : poRing->reversePoints();
1113 : }
1114 77 : bFirstRing = false;
1115 : }
1116 : }
1117 1302 : else if (eFlattenType == wkbMultiPolygon ||
1118 : eFlattenType == wkbGeometryCollection)
1119 : {
1120 35 : for (auto poSubGeom : poGeom->toGeometryCollection())
1121 : {
1122 21 : FixupGeometryBeforeWriting(poSubGeom);
1123 : }
1124 : }
1125 : }
1126 :
1127 : /************************************************************************/
1128 : /* WriteArrowBatch() */
1129 : /************************************************************************/
1130 :
1131 : #if PARQUET_VERSION_MAJOR > 10
1132 : inline bool
1133 14 : OGRParquetWriterLayer::WriteArrowBatch(const struct ArrowSchema *schema,
1134 : struct ArrowArray *array,
1135 : CSLConstList papszOptions)
1136 : {
1137 14 : if (m_poTmpGPKGLayer)
1138 : {
1139 : // When using SORT_BY_BBOX=YES option, we can't directly write the
1140 : // input array, because we need to sort features. Hence we fallback
1141 : // to the OGRLayer base implementation, which will ultimately call
1142 : // OGRParquetWriterLayer::ICreateFeature()
1143 0 : return OGRLayer::WriteArrowBatch(schema, array, papszOptions);
1144 : }
1145 :
1146 28 : return WriteArrowBatchInternal(
1147 : schema, array, papszOptions,
1148 28 : [this](const std::shared_ptr<arrow::RecordBatch> &poBatch)
1149 : {
1150 28 : auto status = m_poFileWriter->NewBufferedRowGroup();
1151 14 : if (!status.ok())
1152 : {
1153 0 : CPLError(CE_Failure, CPLE_AppDefined,
1154 : "NewBufferedRowGroup() failed with %s",
1155 0 : status.message().c_str());
1156 0 : return false;
1157 : }
1158 :
1159 14 : status = m_poFileWriter->WriteRecordBatch(*poBatch);
1160 14 : if (!status.ok())
1161 : {
1162 0 : CPLError(CE_Failure, CPLE_AppDefined,
1163 : "WriteRecordBatch() failed: %s",
1164 0 : status.message().c_str());
1165 0 : return false;
1166 : }
1167 :
1168 14 : return true;
1169 14 : });
1170 : }
1171 : #endif
1172 :
1173 : /************************************************************************/
1174 : /* TestCapability() */
1175 : /************************************************************************/
1176 :
1177 532 : inline int OGRParquetWriterLayer::TestCapability(const char *pszCap) const
1178 : {
1179 : #if PARQUET_VERSION_MAJOR <= 10
1180 : if (EQUAL(pszCap, OLCFastWriteArrowBatch))
1181 : return false;
1182 : #endif
1183 :
1184 532 : if (m_poTmpGPKGLayer && EQUAL(pszCap, OLCFastWriteArrowBatch))
1185 : {
1186 : // When using SORT_BY_BBOX=YES option, we can't directly write the
1187 : // input array, because we need to sort features. So this is not
1188 : // fast
1189 1 : return false;
1190 : }
1191 :
1192 531 : return OGRArrowWriterLayer::TestCapability(pszCap);
1193 : }
1194 :
1195 : /************************************************************************/
1196 : /* CreateFieldFromArrowSchema() */
1197 : /************************************************************************/
1198 :
1199 : #if PARQUET_VERSION_MAJOR > 10
1200 396 : bool OGRParquetWriterLayer::CreateFieldFromArrowSchema(
1201 : const struct ArrowSchema *schema, CSLConstList papszOptions)
1202 : {
1203 396 : if (m_poTmpGPKGLayer)
1204 : {
1205 : // When using SORT_BY_BBOX=YES option, we can't directly write the
1206 : // input array, because we need to sort features. But this process
1207 : // only supports the base Arrow types supported by
1208 : // OGRLayer::WriteArrowBatch()
1209 0 : return OGRLayer::CreateFieldFromArrowSchema(schema, papszOptions);
1210 : }
1211 :
1212 396 : return OGRArrowWriterLayer::CreateFieldFromArrowSchema(schema,
1213 396 : papszOptions);
1214 : }
1215 : #endif
1216 :
1217 : /************************************************************************/
1218 : /* IsArrowSchemaSupported() */
1219 : /************************************************************************/
1220 :
1221 : #if PARQUET_VERSION_MAJOR > 10
1222 1077 : bool OGRParquetWriterLayer::IsArrowSchemaSupported(
1223 : const struct ArrowSchema *schema, CSLConstList papszOptions,
1224 : std::string &osErrorMsg) const
1225 : {
1226 1077 : if (m_poTmpGPKGLayer)
1227 : {
1228 : // When using SORT_BY_BBOX=YES option, we can't directly write the
1229 : // input array, because we need to sort features. But this process
1230 : // only supports the base Arrow types supported by
1231 : // OGRLayer::WriteArrowBatch()
1232 0 : return OGRLayer::IsArrowSchemaSupported(schema, papszOptions,
1233 0 : osErrorMsg);
1234 : }
1235 :
1236 1077 : if (schema->format[0] == 'e' && schema->format[1] == 0)
1237 : {
1238 1 : osErrorMsg = "float16 not supported";
1239 1 : return false;
1240 : }
1241 1076 : if (schema->format[0] == 'v' && schema->format[1] == 'u')
1242 : {
1243 1 : osErrorMsg = "StringView not supported";
1244 1 : return false;
1245 : }
1246 1075 : if (schema->format[0] == 'v' && schema->format[1] == 'z')
1247 : {
1248 1 : osErrorMsg = "BinaryView not supported";
1249 1 : return false;
1250 : }
1251 1074 : if (schema->format[0] == '+' && schema->format[1] == 'v')
1252 : {
1253 0 : if (schema->format[2] == 'l')
1254 : {
1255 0 : osErrorMsg = "ListView not supported";
1256 0 : return false;
1257 : }
1258 0 : else if (schema->format[2] == 'L')
1259 : {
1260 0 : osErrorMsg = "LargeListView not supported";
1261 0 : return false;
1262 : }
1263 : }
1264 2136 : for (int64_t i = 0; i < schema->n_children; ++i)
1265 : {
1266 1065 : if (!IsArrowSchemaSupported(schema->children[i], papszOptions,
1267 : osErrorMsg))
1268 : {
1269 3 : return false;
1270 : }
1271 : }
1272 1071 : return true;
1273 : }
1274 : #endif
1275 :
1276 : /************************************************************************/
1277 : /* SetMetadata() */
1278 : /************************************************************************/
1279 :
1280 13 : CPLErr OGRParquetWriterLayer::SetMetadata(char **papszMetadata,
1281 : const char *pszDomain)
1282 : {
1283 13 : if (!pszDomain || !EQUAL(pszDomain, "SHAPEFILE"))
1284 : {
1285 8 : return OGRLayer::SetMetadata(papszMetadata, pszDomain);
1286 : }
1287 5 : return CE_None;
1288 : }
1289 :
1290 : /************************************************************************/
1291 : /* GetDataset() */
1292 : /************************************************************************/
1293 :
1294 23 : GDALDataset *OGRParquetWriterLayer::GetDataset()
1295 : {
1296 23 : return m_poDataset;
1297 : }
|
