Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: Parquet Translator
4 : * Purpose: Implements OGRParquetDriver.
5 : * Author: Even Rouault, <even.rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2022, Planet Labs
9 : *
10 : * SPDX-License-Identifier: MIT
11 : ****************************************************************************/
12 :
13 : #ifdef STANDALONE
14 : #include "gdal_version.h"
15 : #else
16 : #undef DO_NOT_DEFINE_GDAL_DATE_NAME
17 : #include "gdal_version_full/gdal_version.h"
18 : #endif
19 :
20 : #include "ogr_parquet.h"
21 :
22 : #include "../arrow_common/ograrrowwriterlayer.hpp"
23 :
24 : #include "ogr_wkb.h"
25 :
26 : #include <cassert>
27 : #include <utility>
28 :
29 : /************************************************************************/
30 : /* OGRParquetWriterLayer() */
31 : /************************************************************************/
32 :
33 356 : OGRParquetWriterLayer::OGRParquetWriterLayer(
34 : OGRParquetWriterDataset *poDataset, arrow::MemoryPool *poMemoryPool,
35 : const std::shared_ptr<arrow::io::OutputStream> &poOutputStream,
36 356 : const char *pszLayerName)
37 : : OGRArrowWriterLayer(poMemoryPool, poOutputStream, pszLayerName),
38 356 : m_poDataset(poDataset)
39 : {
40 356 : m_bWriteFieldArrowExtensionName = CPLTestBool(
41 : CPLGetConfigOption("OGR_PARQUET_WRITE_ARROW_EXTENSION_NAME", "NO"));
42 356 : }
43 :
44 : /************************************************************************/
45 : /* Close() */
46 : /************************************************************************/
47 :
48 353 : bool OGRParquetWriterLayer::Close()
49 : {
50 353 : if (m_poTmpGPKGLayer)
51 : {
52 3 : if (!CopyTmpGpkgLayerToFinalFile())
53 0 : return false;
54 : }
55 :
56 353 : if (m_bInitializationOK)
57 : {
58 353 : if (!FinalizeWriting())
59 0 : return false;
60 : }
61 :
62 353 : return true;
63 : }
64 :
65 : /************************************************************************/
66 : /* CopyTmpGpkgLayerToFinalFile() */
67 : /************************************************************************/
68 :
69 3 : bool OGRParquetWriterLayer::CopyTmpGpkgLayerToFinalFile()
70 : {
71 3 : if (!m_poTmpGPKGLayer)
72 : {
73 0 : return true;
74 : }
75 :
76 3 : CPLDebug("PARQUET", "CopyTmpGpkgLayerToFinalFile(): start...");
77 :
78 3 : VSIUnlink(m_poTmpGPKG->GetDescription());
79 :
80 6 : OGRFeature oFeat(m_poFeatureDefn);
81 :
82 : // Interval in terms of features between 2 debug progress report messages
83 3 : constexpr int PROGRESS_FC_INTERVAL = 100 * 1000;
84 :
85 : // First, write features without geometries
86 : {
87 3 : auto poTmpLayer = std::unique_ptr<OGRLayer>(m_poTmpGPKG->ExecuteSQL(
88 : "SELECT serialized_feature FROM tmp WHERE fid NOT IN (SELECT id "
89 : "FROM rtree_tmp_geom)",
90 3 : nullptr, nullptr));
91 3 : if (!poTmpLayer)
92 0 : return false;
93 1005 : for (const auto &poSrcFeature : poTmpLayer.get())
94 : {
95 1002 : int nBytesFeature = 0;
96 : const GByte *pabyFeatureData =
97 1002 : poSrcFeature->GetFieldAsBinary(0, &nBytesFeature);
98 1002 : if (!oFeat.DeserializeFromBinary(pabyFeatureData, nBytesFeature))
99 : {
100 0 : CPLError(CE_Failure, CPLE_AppDefined,
101 : "Cannot deserialize feature");
102 0 : return false;
103 : }
104 1002 : if (OGRArrowWriterLayer::ICreateFeature(&oFeat) != OGRERR_NONE)
105 : {
106 0 : return false;
107 : }
108 :
109 1002 : if ((m_nFeatureCount % PROGRESS_FC_INTERVAL) == 0)
110 : {
111 0 : CPLDebugProgress(
112 : "PARQUET",
113 : "CopyTmpGpkgLayerToFinalFile(): %.02f%% progress",
114 0 : 100.0 * double(m_nFeatureCount) /
115 0 : double(m_nTmpFeatureCount));
116 : }
117 : }
118 :
119 3 : if (!FlushFeatures())
120 : {
121 0 : return false;
122 : }
123 : }
124 :
125 : // Now walk through the GPKG RTree for features with geometries
126 : // Cf https://github.com/sqlite/sqlite/blob/master/ext/rtree/rtree.c
127 : // for the description of the content of the rtree _node table
128 6 : std::vector<std::pair<int64_t, int>> aNodeNoDepthPair;
129 3 : int nTreeDepth = 0;
130 : // Queue the root node
131 : aNodeNoDepthPair.emplace_back(
132 3 : std::make_pair(/* nodeNo = */ 1, /* depth = */ 0));
133 3 : int nCountWrittenFeaturesSinceLastFlush = 0;
134 50 : while (!aNodeNoDepthPair.empty())
135 : {
136 47 : const auto &oLastPair = aNodeNoDepthPair.back();
137 47 : const int64_t nNodeNo = oLastPair.first;
138 47 : const int nCurDepth = oLastPair.second;
139 : //CPLDebug("PARQUET", "Reading nodeNode=%d, curDepth=%d", int(nNodeNo), nCurDepth);
140 47 : aNodeNoDepthPair.pop_back();
141 :
142 47 : auto poRTreeLayer = std::unique_ptr<OGRLayer>(m_poTmpGPKG->ExecuteSQL(
143 : CPLSPrintf("SELECT data FROM rtree_tmp_geom_node WHERE nodeno "
144 : "= " CPL_FRMT_GIB,
145 : static_cast<GIntBig>(nNodeNo)),
146 47 : nullptr, nullptr));
147 47 : if (!poRTreeLayer)
148 : {
149 0 : CPLError(CE_Failure, CPLE_AppDefined,
150 : "Cannot read node " CPL_FRMT_GIB,
151 : static_cast<GIntBig>(nNodeNo));
152 0 : return false;
153 : }
154 : const auto poRTreeFeature =
155 47 : std::unique_ptr<const OGRFeature>(poRTreeLayer->GetNextFeature());
156 47 : if (!poRTreeFeature)
157 : {
158 0 : CPLError(CE_Failure, CPLE_AppDefined,
159 : "Cannot read node " CPL_FRMT_GIB,
160 : static_cast<GIntBig>(nNodeNo));
161 0 : return false;
162 : }
163 :
164 47 : int nNodeBytes = 0;
165 : const GByte *pabyNodeData =
166 47 : poRTreeFeature->GetFieldAsBinary(0, &nNodeBytes);
167 47 : constexpr int BLOB_HEADER_SIZE = 4;
168 47 : if (nNodeBytes < BLOB_HEADER_SIZE)
169 : {
170 0 : CPLError(CE_Failure, CPLE_AppDefined,
171 : "Not enough bytes when reading node " CPL_FRMT_GIB,
172 : static_cast<GIntBig>(nNodeNo));
173 0 : return false;
174 : }
175 47 : if (nNodeNo == 1)
176 : {
177 : // Get the RTree depth from the root node
178 3 : nTreeDepth = (pabyNodeData[0] << 8) | pabyNodeData[1];
179 : //CPLDebug("PARQUET", "nTreeDepth = %d", nTreeDepth);
180 : }
181 :
182 47 : const int nCellCount = (pabyNodeData[2] << 8) | pabyNodeData[3];
183 47 : constexpr int SIZEOF_CELL = 24; // int64_t + 4 float
184 47 : if (nNodeBytes < BLOB_HEADER_SIZE + SIZEOF_CELL * nCellCount)
185 : {
186 0 : CPLError(CE_Failure, CPLE_AppDefined,
187 : "Not enough bytes when reading node " CPL_FRMT_GIB,
188 : static_cast<GIntBig>(nNodeNo));
189 0 : return false;
190 : }
191 :
192 47 : size_t nOffset = BLOB_HEADER_SIZE;
193 47 : if (nCurDepth == nTreeDepth)
194 : {
195 : // Leaf node: it references feature IDs.
196 :
197 : // If we are about to go above m_nRowGroupSize, flush past
198 : // features now, to improve the spatial compacity of the row group.
199 45 : if (m_nRowGroupSize > nCellCount &&
200 45 : nCountWrittenFeaturesSinceLastFlush + nCellCount >
201 45 : m_nRowGroupSize)
202 : {
203 14 : nCountWrittenFeaturesSinceLastFlush = 0;
204 14 : if (!FlushFeatures())
205 : {
206 0 : return false;
207 : }
208 : }
209 :
210 : // nCellCount shouldn't be over 51 normally, but even 65535
211 : // would be fine...
212 45 : assert(nCellCount <= 65535);
213 1247 : for (int i = 0; i < nCellCount; ++i)
214 : {
215 : int64_t nFID;
216 1202 : memcpy(&nFID, pabyNodeData + nOffset, sizeof(int64_t));
217 1202 : CPL_MSBPTR64(&nFID);
218 :
219 : const auto poSrcFeature = std::unique_ptr<const OGRFeature>(
220 1202 : m_poTmpGPKGLayer->GetFeature(nFID));
221 1202 : if (!poSrcFeature)
222 : {
223 0 : CPLError(CE_Failure, CPLE_AppDefined,
224 : "Cannot get feature " CPL_FRMT_GIB,
225 : static_cast<GIntBig>(nFID));
226 0 : return false;
227 : }
228 :
229 1202 : int nBytesFeature = 0;
230 : const GByte *pabyFeatureData =
231 1202 : poSrcFeature->GetFieldAsBinary(0, &nBytesFeature);
232 1202 : if (!oFeat.DeserializeFromBinary(pabyFeatureData,
233 : nBytesFeature))
234 : {
235 0 : CPLError(CE_Failure, CPLE_AppDefined,
236 : "Cannot deserialize feature");
237 0 : return false;
238 : }
239 1202 : if (OGRArrowWriterLayer::ICreateFeature(&oFeat) != OGRERR_NONE)
240 : {
241 0 : return false;
242 : }
243 :
244 1202 : nOffset += SIZEOF_CELL;
245 :
246 1202 : ++nCountWrittenFeaturesSinceLastFlush;
247 :
248 1202 : if ((m_nFeatureCount % PROGRESS_FC_INTERVAL) == 0 ||
249 1202 : m_nFeatureCount == m_nTmpFeatureCount / 2)
250 : {
251 2 : CPLDebugProgress(
252 : "PARQUET",
253 : "CopyTmpGpkgLayerToFinalFile(): %.02f%% progress",
254 2 : 100.0 * double(m_nFeatureCount) /
255 2 : double(m_nTmpFeatureCount));
256 : }
257 : }
258 : }
259 : else
260 : {
261 : // Non-leaf node: it references child nodes.
262 :
263 : // nCellCount shouldn't be over 51 normally, but even 65535
264 : // would be fine...
265 2 : assert(nCellCount <= 65535);
266 46 : for (int i = 0; i < nCellCount; ++i)
267 : {
268 : int64_t nNode;
269 44 : memcpy(&nNode, pabyNodeData + nOffset, sizeof(int64_t));
270 44 : CPL_MSBPTR64(&nNode);
271 : aNodeNoDepthPair.emplace_back(
272 44 : std::make_pair(nNode, nCurDepth + 1));
273 44 : nOffset += SIZEOF_CELL;
274 : }
275 : }
276 : }
277 :
278 3 : CPLDebug("PARQUET",
279 : "CopyTmpGpkgLayerToFinalFile(): 100%%, successfully finished");
280 3 : return true;
281 : }
282 :
283 : /************************************************************************/
284 : /* IsSupportedGeometryType() */
285 : /************************************************************************/
286 :
287 352 : bool OGRParquetWriterLayer::IsSupportedGeometryType(
288 : OGRwkbGeometryType eGType) const
289 : {
290 352 : const auto eFlattenType = wkbFlatten(eGType);
291 352 : if (!OGR_GT_HasM(eGType) && eFlattenType <= wkbGeometryCollection)
292 : {
293 351 : return true;
294 : }
295 :
296 : const auto osConfigOptionName =
297 3 : "OGR_" + GetDriverUCName() + "_ALLOW_ALL_DIMS";
298 1 : if (CPLTestBool(CPLGetConfigOption(osConfigOptionName.c_str(), "NO")))
299 : {
300 0 : return true;
301 : }
302 :
303 1 : CPLError(CE_Failure, CPLE_NotSupported,
304 : "Only 2D and Z geometry types are supported (unless the "
305 : "%s configuration option is set to YES)",
306 : osConfigOptionName.c_str());
307 1 : return false;
308 : }
309 :
310 : /************************************************************************/
311 : /* SetOptions() */
312 : /************************************************************************/
313 :
314 356 : bool OGRParquetWriterLayer::SetOptions(
315 : const OGRGeomFieldDefn *poSrcGeomFieldDefn, CSLConstList papszOptions)
316 : {
317 356 : m_aosCreationOptions = papszOptions;
318 :
319 356 : const char *pszWriteCoveringBBox = CSLFetchNameValueDef(
320 : papszOptions, "WRITE_COVERING_BBOX",
321 : CPLGetConfigOption("OGR_PARQUET_WRITE_COVERING_BBOX", nullptr));
322 356 : m_bWriteBBoxStruct =
323 356 : pszWriteCoveringBBox == nullptr || CPLTestBool(pszWriteCoveringBBox);
324 :
325 : m_oBBoxStructFieldName =
326 356 : CSLFetchNameValueDef(papszOptions, "COVERING_BBOX_NAME", "");
327 :
328 356 : if (CPLTestBool(CSLFetchNameValueDef(papszOptions, "SORT_BY_BBOX", "NO")))
329 : {
330 8 : const std::string osTmpGPKG(std::string(m_poDataset->GetDescription()) +
331 4 : ".tmp.gpkg");
332 4 : auto poGPKGDrv = GetGDALDriverManager()->GetDriverByName("GPKG");
333 4 : if (!poGPKGDrv)
334 : {
335 1 : CPLError(
336 : CE_Failure, CPLE_AppDefined,
337 : "Driver GPKG required for SORT_BY_BBOX layer creation option");
338 1 : return false;
339 : }
340 3 : m_poTmpGPKG.reset(poGPKGDrv->Create(osTmpGPKG.c_str(), 0, 0, 0,
341 : GDT_Unknown, nullptr));
342 3 : if (!m_poTmpGPKG)
343 0 : return false;
344 3 : m_poTmpGPKG->MarkSuppressOnClose();
345 3 : m_poTmpGPKGLayer = m_poTmpGPKG->CreateLayer("tmp");
346 12 : if (!m_poTmpGPKGLayer ||
347 : // Serialized feature
348 3 : m_poTmpGPKGLayer->CreateField(
349 3 : std::make_unique<OGRFieldDefn>("serialized_feature", OFTBinary)
350 3 : .get()) != OGRERR_NONE ||
351 : // FlushCache is needed to avoid SQLite3 errors on empty layers
352 9 : m_poTmpGPKG->FlushCache() != CE_None ||
353 3 : m_poTmpGPKGLayer->StartTransaction() != OGRERR_NONE)
354 : {
355 0 : return false;
356 : }
357 : }
358 :
359 : const char *pszGeomEncoding =
360 355 : CSLFetchNameValue(papszOptions, "GEOMETRY_ENCODING");
361 355 : m_eGeomEncoding = OGRArrowGeomEncoding::WKB;
362 355 : if (pszGeomEncoding)
363 : {
364 211 : if (EQUAL(pszGeomEncoding, "WKB"))
365 7 : m_eGeomEncoding = OGRArrowGeomEncoding::WKB;
366 204 : else if (EQUAL(pszGeomEncoding, "WKT"))
367 8 : m_eGeomEncoding = OGRArrowGeomEncoding::WKT;
368 196 : else if (EQUAL(pszGeomEncoding, "GEOARROW_INTERLEAVED"))
369 : {
370 28 : CPLErrorOnce(
371 : CE_Warning, CPLE_AppDefined,
372 : "Use of GEOMETRY_ENCODING=GEOARROW_INTERLEAVED is not "
373 : "recommended. "
374 : "GeoParquet 1.1 uses GEOMETRY_ENCODING=GEOARROW (struct) "
375 : "instead.");
376 28 : m_eGeomEncoding = OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC;
377 : }
378 168 : else if (EQUAL(pszGeomEncoding, "GEOARROW") ||
379 0 : EQUAL(pszGeomEncoding, "GEOARROW_STRUCT"))
380 168 : m_eGeomEncoding = OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC;
381 : else
382 : {
383 0 : CPLError(CE_Failure, CPLE_NotSupported,
384 : "Unsupported GEOMETRY_ENCODING = %s", pszGeomEncoding);
385 0 : return false;
386 : }
387 : }
388 :
389 : const char *pszCoordPrecision =
390 355 : CSLFetchNameValue(papszOptions, "COORDINATE_PRECISION");
391 355 : if (pszCoordPrecision)
392 0 : m_nWKTCoordinatePrecision = atoi(pszCoordPrecision);
393 :
394 355 : m_bForceCounterClockwiseOrientation =
395 355 : EQUAL(CSLFetchNameValueDef(papszOptions, "POLYGON_ORIENTATION",
396 : "COUNTERCLOCKWISE"),
397 : "COUNTERCLOCKWISE");
398 :
399 : const auto eGType =
400 355 : poSrcGeomFieldDefn ? poSrcGeomFieldDefn->GetType() : wkbNone;
401 355 : if (poSrcGeomFieldDefn && eGType != wkbNone)
402 : {
403 326 : if (!IsSupportedGeometryType(eGType))
404 : {
405 1 : return false;
406 : }
407 :
408 325 : m_poFeatureDefn->SetGeomType(eGType);
409 325 : auto eGeomEncoding = m_eGeomEncoding;
410 325 : if (eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC ||
411 297 : eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC)
412 : {
413 196 : const auto eEncodingType = eGeomEncoding;
414 196 : eGeomEncoding = GetPreciseArrowGeomEncoding(eEncodingType, eGType);
415 196 : if (eGeomEncoding == eEncodingType)
416 0 : return false;
417 : }
418 325 : m_aeGeomEncoding.push_back(eGeomEncoding);
419 :
420 650 : std::string osGeometryName;
421 : const char *pszGeometryName =
422 325 : CSLFetchNameValue(papszOptions, "GEOMETRY_NAME");
423 325 : if (pszGeometryName)
424 16 : osGeometryName = pszGeometryName;
425 309 : else if (poSrcGeomFieldDefn->GetNameRef()[0])
426 4 : osGeometryName = poSrcGeomFieldDefn->GetNameRef();
427 : else
428 305 : osGeometryName = "geometry";
429 325 : m_poFeatureDefn->GetGeomFieldDefn(0)->SetName(osGeometryName.c_str());
430 :
431 325 : const auto poSpatialRef = poSrcGeomFieldDefn->GetSpatialRef();
432 325 : if (poSpatialRef)
433 : {
434 42 : auto poSRS = poSpatialRef->Clone();
435 42 : m_poFeatureDefn->GetGeomFieldDefn(0)->SetSpatialRef(poSRS);
436 42 : poSRS->Release();
437 : }
438 : }
439 :
440 354 : m_osFIDColumn = CSLFetchNameValueDef(papszOptions, "FID", "");
441 :
442 354 : const char *pszCompression = CSLFetchNameValue(papszOptions, "COMPRESSION");
443 354 : if (pszCompression == nullptr)
444 : {
445 1044 : auto oResult = arrow::util::Codec::GetCompressionType("snappy");
446 348 : if (oResult.ok() && arrow::util::Codec::IsAvailable(*oResult))
447 : {
448 348 : pszCompression = "SNAPPY";
449 : }
450 : else
451 : {
452 0 : pszCompression = "NONE";
453 : }
454 : }
455 :
456 354 : if (EQUAL(pszCompression, "NONE"))
457 0 : pszCompression = "UNCOMPRESSED";
458 : auto oResult = arrow::util::Codec::GetCompressionType(
459 708 : CPLString(pszCompression).tolower());
460 354 : if (!oResult.ok())
461 : {
462 1 : CPLError(CE_Failure, CPLE_NotSupported,
463 : "Unrecognized compression method: %s", pszCompression);
464 1 : return false;
465 : }
466 353 : m_eCompression = *oResult;
467 353 : if (!arrow::util::Codec::IsAvailable(m_eCompression))
468 : {
469 0 : CPLError(CE_Failure, CPLE_NotSupported,
470 : "Compression method %s is known, but libarrow has not "
471 : "been built with support for it",
472 : pszCompression);
473 0 : return false;
474 : }
475 353 : m_oWriterPropertiesBuilder.compression(m_eCompression);
476 :
477 : const char *pszCompressionLevel =
478 353 : CSLFetchNameValue(papszOptions, "COMPRESSION_LEVEL");
479 353 : if (pszCompressionLevel)
480 : {
481 2 : const int nCompressionLevel = atoi(pszCompressionLevel);
482 2 : if (nCompressionLevel != DEFAULT_COMPRESSION_LEVEL)
483 2 : m_oWriterPropertiesBuilder.compression_level(nCompressionLevel);
484 : }
485 351 : else if (EQUAL(pszCompression, "ZSTD"))
486 1 : m_oWriterPropertiesBuilder.compression_level(
487 : OGR_PARQUET_ZSTD_DEFAULT_COMPRESSION_LEVEL);
488 :
489 : const std::string osCreator =
490 353 : CSLFetchNameValueDef(papszOptions, "CREATOR", "");
491 353 : if (!osCreator.empty())
492 1 : m_oWriterPropertiesBuilder.created_by(osCreator);
493 : else
494 352 : m_oWriterPropertiesBuilder.created_by("GDAL " GDAL_RELEASE_NAME
495 : ", using " CREATED_BY_VERSION);
496 :
497 : // Undocumented option. Not clear it is useful besides unit test purposes
498 353 : if (!CPLTestBool(CSLFetchNameValueDef(papszOptions, "STATISTICS", "YES")))
499 1 : m_oWriterPropertiesBuilder.disable_statistics();
500 :
501 : #if PARQUET_VERSION_MAJOR >= 12
502 : // Undocumented option. Not clear it is useful to disable it.
503 353 : if (CPLTestBool(CSLFetchNameValueDef(papszOptions, "PAGE_INDEX", "YES")))
504 353 : m_oWriterPropertiesBuilder.enable_write_page_index();
505 : #endif
506 :
507 : const char *pszWriteGeo =
508 353 : CPLGetConfigOption("OGR_PARQUET_WRITE_GEO", nullptr);
509 353 : m_bWriteGeoMetadata = pszWriteGeo == nullptr || CPLTestBool(pszWriteGeo);
510 :
511 353 : if (m_eGeomEncoding == OGRArrowGeomEncoding::WKB && eGType != wkbNone)
512 : {
513 : #if ARROW_VERSION_MAJOR >= 21
514 : const char *pszUseParquetGeoTypes =
515 : CSLFetchNameValueDef(papszOptions, "USE_PARQUET_GEO_TYPES", "NO");
516 : if (EQUAL(pszUseParquetGeoTypes, "ONLY"))
517 : {
518 : m_bUseArrowWKBExtension = true;
519 : if (pszWriteGeo == nullptr)
520 : m_bWriteGeoMetadata = false;
521 : if (pszWriteCoveringBBox == nullptr)
522 : m_bWriteBBoxStruct = false;
523 : }
524 : else
525 : {
526 : m_bUseArrowWKBExtension = CPLTestBool(pszUseParquetGeoTypes);
527 : }
528 : #else
529 128 : m_oWriterPropertiesBuilder.disable_statistics(
530 384 : parquet::schema::ColumnPath::FromDotString(
531 128 : m_poFeatureDefn->GetGeomFieldDefn(0)->GetNameRef()));
532 : #endif
533 : }
534 :
535 : const char *pszRowGroupSize =
536 353 : CSLFetchNameValue(papszOptions, "ROW_GROUP_SIZE");
537 353 : if (pszRowGroupSize)
538 : {
539 19 : auto nRowGroupSize = static_cast<int64_t>(atoll(pszRowGroupSize));
540 19 : if (nRowGroupSize > 0)
541 : {
542 19 : if (nRowGroupSize > INT_MAX)
543 0 : nRowGroupSize = INT_MAX;
544 19 : m_nRowGroupSize = nRowGroupSize;
545 : }
546 : }
547 :
548 353 : m_bEdgesSpherical = EQUAL(
549 : CSLFetchNameValueDef(papszOptions, "EDGES", "PLANAR"), "SPHERICAL");
550 :
551 353 : m_bInitializationOK = true;
552 353 : return true;
553 : }
554 :
555 : /************************************************************************/
556 : /* CloseFileWriter() */
557 : /************************************************************************/
558 :
559 353 : bool OGRParquetWriterLayer::CloseFileWriter()
560 : {
561 706 : auto status = m_poFileWriter->Close();
562 353 : if (!status.ok())
563 : {
564 0 : CPLError(CE_Failure, CPLE_AppDefined,
565 : "FileWriter::Close() failed with %s",
566 0 : status.message().c_str());
567 : }
568 706 : return status.ok();
569 : }
570 :
571 : /************************************************************************/
572 : /* GetGeoMetadata() */
573 : /************************************************************************/
574 :
575 353 : std::string OGRParquetWriterLayer::GetGeoMetadata() const
576 : {
577 : // Just for unit testing purposes
578 : const char *pszGeoMetadata =
579 353 : CPLGetConfigOption("OGR_PARQUET_GEO_METADATA", nullptr);
580 353 : if (pszGeoMetadata)
581 16 : return pszGeoMetadata;
582 :
583 337 : if (m_poFeatureDefn->GetGeomFieldCount() != 0 && m_bWriteGeoMetadata)
584 : {
585 634 : CPLJSONObject oRoot;
586 317 : oRoot.Add("version", "1.1.0");
587 317 : oRoot.Add("primary_column",
588 317 : m_poFeatureDefn->GetGeomFieldDefn(0)->GetNameRef());
589 634 : CPLJSONObject oColumns;
590 317 : oRoot.Add("columns", oColumns);
591 651 : for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
592 : {
593 334 : const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
594 668 : CPLJSONObject oColumn;
595 334 : oColumns.Add(poGeomFieldDefn->GetNameRef(), oColumn);
596 334 : oColumn.Add("encoding",
597 334 : GetGeomEncodingAsString(m_aeGeomEncoding[i], true));
598 :
599 334 : if (CPLTestBool(CPLGetConfigOption("OGR_PARQUET_WRITE_CRS", "YES")))
600 : {
601 333 : const auto poSRS = poGeomFieldDefn->GetSpatialRef();
602 333 : if (poSRS)
603 : {
604 82 : OGRSpatialReference oSRSIdentified(IdentifyCRS(poSRS));
605 :
606 : const char *pszAuthName =
607 41 : oSRSIdentified.GetAuthorityName(nullptr);
608 : const char *pszAuthCode =
609 41 : oSRSIdentified.GetAuthorityCode(nullptr);
610 :
611 41 : bool bOmitCRS = false;
612 41 : if (pszAuthName != nullptr && pszAuthCode != nullptr &&
613 40 : ((EQUAL(pszAuthName, "EPSG") &&
614 37 : EQUAL(pszAuthCode, "4326")) ||
615 22 : (EQUAL(pszAuthName, "OGC") &&
616 3 : EQUAL(pszAuthCode, "CRS84"))))
617 : {
618 : // To make things less confusing for non-geo-aware
619 : // consumers, omit EPSG:4326 / OGC:CRS84 CRS by default
620 21 : bOmitCRS = CPLTestBool(CPLGetConfigOption(
621 : "OGR_PARQUET_CRS_OMIT_IF_WGS84", "YES"));
622 : }
623 :
624 41 : if (bOmitCRS)
625 : {
626 : // do nothing
627 : }
628 20 : else if (EQUAL(CPLGetConfigOption(
629 : "OGR_PARQUET_CRS_ENCODING", "PROJJSON"),
630 : "PROJJSON"))
631 : {
632 : // CRS encoded as PROJJSON for GeoParquet >= 0.4.0
633 20 : char *pszPROJJSON = nullptr;
634 20 : oSRSIdentified.exportToPROJJSON(&pszPROJJSON, nullptr);
635 40 : CPLJSONDocument oCRSDoc;
636 20 : CPL_IGNORE_RET_VAL(oCRSDoc.LoadMemory(pszPROJJSON));
637 20 : CPLFree(pszPROJJSON);
638 20 : CPLJSONObject oCRSRoot = oCRSDoc.GetRoot();
639 20 : RemoveIDFromMemberOfEnsembles(oCRSRoot);
640 20 : oColumn.Add("crs", oCRSRoot);
641 : }
642 : else
643 : {
644 : // WKT was used in GeoParquet <= 0.3.0
645 0 : const char *const apszOptions[] = {
646 : "FORMAT=WKT2_2019", "MULTILINE=NO", nullptr};
647 0 : char *pszWKT = nullptr;
648 0 : oSRSIdentified.exportToWkt(&pszWKT, apszOptions);
649 0 : if (pszWKT)
650 0 : oColumn.Add("crs", pszWKT);
651 0 : CPLFree(pszWKT);
652 : }
653 :
654 41 : const double dfCoordEpoch = poSRS->GetCoordinateEpoch();
655 41 : if (dfCoordEpoch > 0)
656 2 : oColumn.Add("epoch", dfCoordEpoch);
657 : }
658 : else
659 : {
660 292 : oColumn.AddNull("crs");
661 : }
662 : }
663 :
664 334 : if (m_bEdgesSpherical)
665 : {
666 3 : oColumn.Add("edges", "spherical");
667 : }
668 :
669 635 : if (m_aoEnvelopes[i].IsInit() &&
670 301 : CPLTestBool(
671 : CPLGetConfigOption("OGR_PARQUET_WRITE_BBOX", "YES")))
672 : {
673 301 : bool bHasZ = false;
674 537 : for (const auto eGeomType : m_oSetWrittenGeometryTypes[i])
675 : {
676 343 : bHasZ = OGR_GT_HasZ(eGeomType);
677 343 : if (bHasZ)
678 107 : break;
679 : }
680 301 : CPLJSONArray oBBOX;
681 301 : oBBOX.Add(m_aoEnvelopes[i].MinX);
682 301 : oBBOX.Add(m_aoEnvelopes[i].MinY);
683 301 : if (bHasZ)
684 107 : oBBOX.Add(m_aoEnvelopes[i].MinZ);
685 301 : oBBOX.Add(m_aoEnvelopes[i].MaxX);
686 301 : oBBOX.Add(m_aoEnvelopes[i].MaxY);
687 301 : if (bHasZ)
688 107 : oBBOX.Add(m_aoEnvelopes[i].MaxZ);
689 301 : oColumn.Add("bbox", oBBOX);
690 : }
691 :
692 : // Bounding box column definition
693 598 : if (m_bWriteBBoxStruct &&
694 264 : CPLTestBool(CPLGetConfigOption(
695 : "OGR_PARQUET_WRITE_COVERING_BBOX_IN_METADATA", "YES")))
696 : {
697 528 : CPLJSONObject oCovering;
698 264 : oColumn.Add("covering", oCovering);
699 528 : CPLJSONObject oBBOX;
700 264 : oCovering.Add("bbox", oBBOX);
701 : const auto AddComponent =
702 3168 : [this, i, &oBBOX](const char *pszComponent)
703 : {
704 1056 : CPLJSONArray oArray;
705 1056 : oArray.Add(m_apoFieldsBBOX[i]->name());
706 1056 : oArray.Add(pszComponent);
707 1056 : oBBOX.Add(pszComponent, oArray);
708 1056 : };
709 264 : AddComponent("xmin");
710 264 : AddComponent("ymin");
711 264 : AddComponent("xmax");
712 264 : AddComponent("ymax");
713 : }
714 :
715 359 : const auto GetStringGeometryType = [](OGRwkbGeometryType eType)
716 : {
717 359 : const auto eFlattenType = wkbFlatten(eType);
718 359 : std::string osType = "Unknown";
719 359 : if (wkbPoint == eFlattenType)
720 85 : osType = "Point";
721 274 : else if (wkbLineString == eFlattenType)
722 42 : osType = "LineString";
723 232 : else if (wkbPolygon == eFlattenType)
724 75 : osType = "Polygon";
725 157 : else if (wkbMultiPoint == eFlattenType)
726 34 : osType = "MultiPoint";
727 123 : else if (wkbMultiLineString == eFlattenType)
728 37 : osType = "MultiLineString";
729 86 : else if (wkbMultiPolygon == eFlattenType)
730 81 : osType = "MultiPolygon";
731 5 : else if (wkbGeometryCollection == eFlattenType)
732 5 : osType = "GeometryCollection";
733 359 : if (osType != "Unknown")
734 : {
735 : // M and ZM not supported officially currently, but it
736 : // doesn't hurt to anticipate
737 359 : if (OGR_GT_HasZ(eType) && OGR_GT_HasM(eType))
738 8 : osType += " ZM";
739 351 : else if (OGR_GT_HasZ(eType))
740 115 : osType += " Z";
741 236 : else if (OGR_GT_HasM(eType))
742 8 : osType += " M";
743 : }
744 359 : return osType;
745 : };
746 :
747 334 : if (m_bForceCounterClockwiseOrientation)
748 333 : oColumn.Add("orientation", "counterclockwise");
749 :
750 334 : CPLJSONArray oArray;
751 693 : for (const auto eType : m_oSetWrittenGeometryTypes[i])
752 : {
753 359 : oArray.Add(GetStringGeometryType(eType));
754 : }
755 334 : oColumn.Add("geometry_types", oArray);
756 : }
757 :
758 317 : return oRoot.Format(CPLJSONObject::PrettyFormat::Plain);
759 : }
760 20 : return std::string();
761 : }
762 :
763 : /************************************************************************/
764 : /* PerformStepsBeforeFinalFlushGroup() */
765 : /************************************************************************/
766 :
767 353 : void OGRParquetWriterLayer::PerformStepsBeforeFinalFlushGroup()
768 : {
769 353 : if (m_poKeyValueMetadata)
770 : {
771 706 : std::string osGeoMetadata = GetGeoMetadata();
772 706 : auto poTmpSchema = m_poSchema;
773 353 : if (!osGeoMetadata.empty())
774 : {
775 : // HACK: it would be good for Arrow to provide a clean way to alter
776 : // key value metadata before finalizing.
777 : // We need to write metadata at end to write the bounding box.
778 333 : const_cast<arrow::KeyValueMetadata *>(m_poKeyValueMetadata.get())
779 333 : ->Append("geo", osGeoMetadata);
780 :
781 333 : auto kvMetadata = poTmpSchema->metadata()
782 15 : ? poTmpSchema->metadata()->Copy()
783 348 : : std::make_shared<arrow::KeyValueMetadata>();
784 333 : kvMetadata->Append("geo", std::move(osGeoMetadata));
785 333 : poTmpSchema = poTmpSchema->WithMetadata(kvMetadata);
786 : }
787 :
788 353 : if (CPLTestBool(
789 : CPLGetConfigOption("OGR_PARQUET_WRITE_ARROW_SCHEMA", "YES")))
790 : {
791 : auto status =
792 706 : ::arrow::ipc::SerializeSchema(*poTmpSchema, m_poMemoryPool);
793 353 : if (status.ok())
794 : {
795 : // The serialized schema is not UTF-8, which is required for
796 : // Thrift
797 706 : const std::string schema_as_string = (*status)->ToString();
798 : std::string schema_base64 =
799 353 : ::arrow::util::base64_encode(schema_as_string);
800 353 : static const std::string kArrowSchemaKey = "ARROW:schema";
801 : const_cast<arrow::KeyValueMetadata *>(
802 353 : m_poKeyValueMetadata.get())
803 353 : ->Append(kArrowSchemaKey, std::move(schema_base64));
804 : }
805 : }
806 :
807 : // Put GDAL metadata into a gdal:metadata domain
808 706 : CPLJSONObject oMultiMetadata;
809 353 : bool bHasMultiMetadata = false;
810 360 : auto &l_oMDMD = oMDMD.GetDomainList() && *(oMDMD.GetDomainList())
811 360 : ? oMDMD
812 346 : : m_poDataset->GetMultiDomainMetadata();
813 362 : for (CSLConstList papszDomainIter = l_oMDMD.GetDomainList();
814 362 : papszDomainIter && *papszDomainIter; ++papszDomainIter)
815 : {
816 9 : const char *pszDomain = *papszDomainIter;
817 9 : CSLConstList papszMD = l_oMDMD.GetMetadata(pszDomain);
818 9 : if (STARTS_WITH(pszDomain, "json:") && papszMD && papszMD[0])
819 : {
820 1 : CPLJSONDocument oDoc;
821 1 : if (oDoc.LoadMemory(papszMD[0]))
822 : {
823 1 : bHasMultiMetadata = true;
824 1 : oMultiMetadata.Add(pszDomain, oDoc.GetRoot());
825 1 : continue;
826 0 : }
827 : }
828 8 : else if (STARTS_WITH(pszDomain, "xml:") && papszMD && papszMD[0])
829 : {
830 1 : bHasMultiMetadata = true;
831 1 : oMultiMetadata.Add(pszDomain, papszMD[0]);
832 1 : continue;
833 : }
834 14 : CPLJSONObject oMetadata;
835 7 : bool bHasMetadata = false;
836 14 : for (CSLConstList papszMDIter = papszMD;
837 14 : papszMDIter && *papszMDIter; ++papszMDIter)
838 : {
839 7 : char *pszKey = nullptr;
840 7 : const char *pszValue = CPLParseNameValue(*papszMDIter, &pszKey);
841 7 : if (pszKey && pszValue)
842 : {
843 7 : bHasMetadata = true;
844 7 : bHasMultiMetadata = true;
845 7 : oMetadata.Add(pszKey, pszValue);
846 : }
847 7 : CPLFree(pszKey);
848 : }
849 7 : if (bHasMetadata)
850 7 : oMultiMetadata.Add(pszDomain, oMetadata);
851 : }
852 353 : if (bHasMultiMetadata)
853 : {
854 7 : const_cast<arrow::KeyValueMetadata *>(m_poKeyValueMetadata.get())
855 7 : ->Append(
856 : "gdal:metadata",
857 14 : oMultiMetadata.Format(CPLJSONObject::PrettyFormat::Plain));
858 : }
859 :
860 353 : if (!m_aosCreationOptions.empty())
861 : {
862 496 : CPLJSONObject oCreationOptions;
863 248 : bool bEmpty = true;
864 1108 : for (const auto &[key, value] :
865 1356 : cpl::IterateNameValue(m_aosCreationOptions))
866 : {
867 554 : if (!EQUAL(key, "FID") && !EQUAL(key, "GEOMETRY_NAME") &&
868 515 : !EQUAL(key, "EDGES"))
869 : {
870 511 : bEmpty = false;
871 511 : oCreationOptions.Add(key, value);
872 : }
873 : }
874 248 : if (!bEmpty)
875 : {
876 : const_cast<arrow::KeyValueMetadata *>(
877 234 : m_poKeyValueMetadata.get())
878 234 : ->Append("gdal:creation-options",
879 468 : oCreationOptions.Format(
880 : CPLJSONObject::PrettyFormat::Plain));
881 : }
882 : }
883 : }
884 353 : }
885 :
886 : /************************************************************************/
887 : /* Open() */
888 : /************************************************************************/
889 :
890 : // Same as parquet::arrow::FileWriter::Open(), except we also
891 : // return KeyValueMetadata
892 : static arrow::Status
893 353 : Open(const ::arrow::Schema &schema, ::arrow::MemoryPool *pool,
894 : std::shared_ptr<::arrow::io::OutputStream> sink,
895 : std::shared_ptr<parquet::WriterProperties> properties,
896 : std::shared_ptr<parquet::ArrowWriterProperties> arrow_properties,
897 : std::unique_ptr<parquet::arrow::FileWriter> *writer,
898 : std::shared_ptr<const arrow::KeyValueMetadata> *outMetadata)
899 : {
900 353 : std::shared_ptr<parquet::SchemaDescriptor> parquet_schema;
901 706 : RETURN_NOT_OK(parquet::arrow::ToParquetSchema(
902 : &schema, *properties, *arrow_properties, &parquet_schema));
903 :
904 : auto schema_node = std::static_pointer_cast<parquet::schema::GroupNode>(
905 706 : parquet_schema->schema_root());
906 :
907 353 : auto metadata = schema.metadata()
908 20 : ? schema.metadata()->Copy()
909 726 : : std::make_shared<arrow::KeyValueMetadata>();
910 353 : *outMetadata = metadata;
911 :
912 353 : std::unique_ptr<parquet::ParquetFileWriter> base_writer;
913 353 : PARQUET_CATCH_NOT_OK(base_writer = parquet::ParquetFileWriter::Open(
914 : std::move(sink), std::move(schema_node),
915 : std::move(properties), metadata));
916 :
917 353 : auto schema_ptr = std::make_shared<::arrow::Schema>(schema);
918 : return parquet::arrow::FileWriter::Make(
919 706 : pool, std::move(base_writer), std::move(schema_ptr),
920 1059 : std::move(arrow_properties), writer);
921 : }
922 :
923 : /************************************************************************/
924 : /* CreateSchema() */
925 : /************************************************************************/
926 :
927 353 : void OGRParquetWriterLayer::CreateSchema()
928 : {
929 353 : CreateSchemaCommon();
930 353 : }
931 :
932 : /************************************************************************/
933 : /* CreateGeomField() */
934 : /************************************************************************/
935 :
936 27 : OGRErr OGRParquetWriterLayer::CreateGeomField(const OGRGeomFieldDefn *poField,
937 : int bApproxOK)
938 : {
939 27 : OGRErr eErr = OGRArrowWriterLayer::CreateGeomField(poField, bApproxOK);
940 53 : if (eErr == OGRERR_NONE &&
941 26 : m_aeGeomEncoding.back() == OGRArrowGeomEncoding::WKB
942 : #if ARROW_VERSION_MAJOR < 21
943 : // Geostatistics in Arrow 21 do not support geographic type for now
944 53 : && m_bEdgesSpherical
945 : #endif
946 : )
947 : {
948 0 : m_oWriterPropertiesBuilder.disable_statistics(
949 0 : parquet::schema::ColumnPath::FromDotString(
950 0 : m_poFeatureDefn
951 0 : ->GetGeomFieldDefn(m_poFeatureDefn->GetGeomFieldCount() - 1)
952 : ->GetNameRef()));
953 : }
954 27 : return eErr;
955 : }
956 :
957 : /************************************************************************/
958 : /* CreateWriter() */
959 : /************************************************************************/
960 :
961 353 : void OGRParquetWriterLayer::CreateWriter()
962 : {
963 353 : CPLAssert(m_poFileWriter == nullptr);
964 :
965 353 : if (m_poSchema == nullptr)
966 : {
967 41 : CreateSchema();
968 : }
969 : else
970 : {
971 312 : FinalizeSchema();
972 : }
973 :
974 : auto arrowWriterProperties =
975 353 : parquet::ArrowWriterProperties::Builder().store_schema()->build();
976 1059 : CPL_IGNORE_RET_VAL(Open(*m_poSchema, m_poMemoryPool, m_poOutputStream,
977 706 : m_oWriterPropertiesBuilder.build(),
978 353 : std::move(arrowWriterProperties), &m_poFileWriter,
979 : &m_poKeyValueMetadata));
980 353 : }
981 :
982 : /************************************************************************/
983 : /* ICreateFeature() */
984 : /************************************************************************/
985 :
986 3378 : OGRErr OGRParquetWriterLayer::ICreateFeature(OGRFeature *poFeature)
987 : {
988 : // If not using SORT_BY_BBOX=YES layer creation option, we can directly
989 : // write features to the final Parquet file
990 3378 : if (!m_poTmpGPKGLayer)
991 1174 : return OGRArrowWriterLayer::ICreateFeature(poFeature);
992 :
993 : // SORT_BY_BBOX=YES case: we write for now a serialized version of poFeature
994 : // in a temporary GeoPackage file.
995 :
996 2204 : GIntBig nFID = poFeature->GetFID();
997 2204 : if (!m_osFIDColumn.empty() && nFID == OGRNullFID)
998 : {
999 1102 : nFID = m_nTmpFeatureCount;
1000 1102 : poFeature->SetFID(nFID);
1001 : }
1002 2204 : ++m_nTmpFeatureCount;
1003 :
1004 4408 : std::vector<GByte> abyBuffer;
1005 : // Serialize the source feature as a single array of bytes to preserve it
1006 : // fully
1007 2204 : if (!poFeature->SerializeToBinary(abyBuffer))
1008 : {
1009 0 : return OGRERR_FAILURE;
1010 : }
1011 :
1012 : // SQLite3 limitation: a row must fit in slightly less than 1 GB.
1013 2204 : constexpr int SOME_MARGIN = 128;
1014 2204 : if (abyBuffer.size() > 1024 * 1024 * 1024 - SOME_MARGIN)
1015 : {
1016 0 : CPLError(CE_Failure, CPLE_NotSupported,
1017 : "Features larger than 1 GB are not supported");
1018 0 : return OGRERR_FAILURE;
1019 : }
1020 :
1021 4408 : OGRFeature oFeat(m_poTmpGPKGLayer->GetLayerDefn());
1022 2204 : oFeat.SetFID(nFID);
1023 2204 : oFeat.SetField(0, static_cast<int>(abyBuffer.size()), abyBuffer.data());
1024 2204 : const auto poSrcGeom = poFeature->GetGeometryRef();
1025 2204 : if (poSrcGeom && !poSrcGeom->IsEmpty())
1026 : {
1027 : // For the purpose of building an RTree, just use the bounding box of
1028 : // the geometry as the geometry.
1029 1202 : OGREnvelope sEnvelope;
1030 1202 : poSrcGeom->getEnvelope(&sEnvelope);
1031 2404 : auto poPoly = std::make_unique<OGRPolygon>();
1032 2404 : auto poLR = std::make_unique<OGRLinearRing>();
1033 1202 : poLR->addPoint(sEnvelope.MinX, sEnvelope.MinY);
1034 1202 : poLR->addPoint(sEnvelope.MinX, sEnvelope.MaxY);
1035 1202 : poLR->addPoint(sEnvelope.MaxX, sEnvelope.MaxY);
1036 1202 : poLR->addPoint(sEnvelope.MaxX, sEnvelope.MinY);
1037 1202 : poLR->addPoint(sEnvelope.MinX, sEnvelope.MinY);
1038 1202 : poPoly->addRingDirectly(poLR.release());
1039 1202 : oFeat.SetGeometryDirectly(poPoly.release());
1040 : }
1041 2204 : return m_poTmpGPKGLayer->CreateFeature(&oFeat);
1042 : }
1043 :
1044 : /************************************************************************/
1045 : /* FlushGroup() */
1046 : /************************************************************************/
1047 :
1048 330 : bool OGRParquetWriterLayer::FlushGroup()
1049 : {
1050 : #if PARQUET_VERSION_MAJOR >= 20
1051 : auto status = m_poFileWriter->NewRowGroup();
1052 : #else
1053 660 : auto status = m_poFileWriter->NewRowGroup(m_apoBuilders[0]->length());
1054 : #endif
1055 330 : if (!status.ok())
1056 : {
1057 0 : CPLError(CE_Failure, CPLE_AppDefined, "NewRowGroup() failed with %s",
1058 0 : status.message().c_str());
1059 0 : ClearArrayBuilers();
1060 0 : return false;
1061 : }
1062 :
1063 330 : auto ret = WriteArrays(
1064 1279 : [this](const std::shared_ptr<arrow::Field> &field,
1065 1279 : const std::shared_ptr<arrow::Array> &array)
1066 : {
1067 2558 : auto l_status = m_poFileWriter->WriteColumnChunk(*array);
1068 1279 : if (!l_status.ok())
1069 : {
1070 0 : CPLError(CE_Failure, CPLE_AppDefined,
1071 : "WriteColumnChunk() failed for field %s: %s",
1072 0 : field->name().c_str(), l_status.message().c_str());
1073 0 : return false;
1074 : }
1075 1279 : return true;
1076 : });
1077 :
1078 330 : ClearArrayBuilers();
1079 330 : return ret;
1080 : }
1081 :
1082 : /************************************************************************/
1083 : /* FixupWKBGeometryBeforeWriting() */
1084 : /************************************************************************/
1085 :
1086 51 : void OGRParquetWriterLayer::FixupWKBGeometryBeforeWriting(GByte *pabyWkb,
1087 : size_t nLen)
1088 : {
1089 51 : if (!m_bForceCounterClockwiseOrientation)
1090 0 : return;
1091 :
1092 51 : OGRWKBFixupCounterClockWiseExternalRing(pabyWkb, nLen);
1093 : }
1094 :
1095 : /************************************************************************/
1096 : /* FixupGeometryBeforeWriting() */
1097 : /************************************************************************/
1098 :
1099 1379 : void OGRParquetWriterLayer::FixupGeometryBeforeWriting(OGRGeometry *poGeom)
1100 : {
1101 1379 : if (!m_bForceCounterClockwiseOrientation)
1102 3 : return;
1103 :
1104 1376 : const auto eFlattenType = wkbFlatten(poGeom->getGeometryType());
1105 : // Polygon rings MUST follow the right-hand rule for orientation
1106 : // (counterclockwise external rings, clockwise internal rings)
1107 1376 : if (eFlattenType == wkbPolygon)
1108 : {
1109 74 : bool bFirstRing = true;
1110 151 : for (auto poRing : poGeom->toPolygon())
1111 : {
1112 85 : if ((bFirstRing && poRing->isClockwise()) ||
1113 8 : (!bFirstRing && !poRing->isClockwise()))
1114 : {
1115 72 : poRing->reversePoints();
1116 : }
1117 77 : bFirstRing = false;
1118 : }
1119 : }
1120 1302 : else if (eFlattenType == wkbMultiPolygon ||
1121 : eFlattenType == wkbGeometryCollection)
1122 : {
1123 35 : for (auto poSubGeom : poGeom->toGeometryCollection())
1124 : {
1125 21 : FixupGeometryBeforeWriting(poSubGeom);
1126 : }
1127 : }
1128 : }
1129 :
1130 : /************************************************************************/
1131 : /* WriteArrowBatch() */
1132 : /************************************************************************/
1133 :
1134 : #if PARQUET_VERSION_MAJOR > 10
1135 : inline bool
1136 22 : OGRParquetWriterLayer::WriteArrowBatch(const struct ArrowSchema *schema,
1137 : struct ArrowArray *array,
1138 : CSLConstList papszOptions)
1139 : {
1140 22 : if (m_poTmpGPKGLayer)
1141 : {
1142 : // When using SORT_BY_BBOX=YES option, we can't directly write the
1143 : // input array, because we need to sort features. Hence we fallback
1144 : // to the OGRLayer base implementation, which will ultimately call
1145 : // OGRParquetWriterLayer::ICreateFeature()
1146 0 : return OGRLayer::WriteArrowBatch(schema, array, papszOptions);
1147 : }
1148 :
1149 44 : return WriteArrowBatchInternal(
1150 : schema, array, papszOptions,
1151 44 : [this](const std::shared_ptr<arrow::RecordBatch> &poBatch)
1152 : {
1153 44 : auto status = m_poFileWriter->NewBufferedRowGroup();
1154 22 : if (!status.ok())
1155 : {
1156 0 : CPLError(CE_Failure, CPLE_AppDefined,
1157 : "NewBufferedRowGroup() failed with %s",
1158 0 : status.message().c_str());
1159 0 : return false;
1160 : }
1161 :
1162 22 : status = m_poFileWriter->WriteRecordBatch(*poBatch);
1163 22 : if (!status.ok())
1164 : {
1165 0 : CPLError(CE_Failure, CPLE_AppDefined,
1166 : "WriteRecordBatch() failed: %s",
1167 0 : status.message().c_str());
1168 0 : return false;
1169 : }
1170 :
1171 22 : return true;
1172 22 : });
1173 : }
1174 : #endif
1175 :
1176 : /************************************************************************/
1177 : /* TestCapability() */
1178 : /************************************************************************/
1179 :
1180 644 : inline int OGRParquetWriterLayer::TestCapability(const char *pszCap) const
1181 : {
1182 : #if PARQUET_VERSION_MAJOR <= 10
1183 : if (EQUAL(pszCap, OLCFastWriteArrowBatch))
1184 : return false;
1185 : #endif
1186 :
1187 644 : if (m_poTmpGPKGLayer && EQUAL(pszCap, OLCFastWriteArrowBatch))
1188 : {
1189 : // When using SORT_BY_BBOX=YES option, we can't directly write the
1190 : // input array, because we need to sort features. So this is not
1191 : // fast
1192 1 : return false;
1193 : }
1194 :
1195 643 : return OGRArrowWriterLayer::TestCapability(pszCap);
1196 : }
1197 :
1198 : /************************************************************************/
1199 : /* CreateFieldFromArrowSchema() */
1200 : /************************************************************************/
1201 :
1202 : #if PARQUET_VERSION_MAJOR > 10
1203 473 : bool OGRParquetWriterLayer::CreateFieldFromArrowSchema(
1204 : const struct ArrowSchema *schema, CSLConstList papszOptions)
1205 : {
1206 473 : if (m_poTmpGPKGLayer)
1207 : {
1208 : // When using SORT_BY_BBOX=YES option, we can't directly write the
1209 : // input array, because we need to sort features. But this process
1210 : // only supports the base Arrow types supported by
1211 : // OGRLayer::WriteArrowBatch()
1212 0 : return OGRLayer::CreateFieldFromArrowSchema(schema, papszOptions);
1213 : }
1214 :
1215 473 : return OGRArrowWriterLayer::CreateFieldFromArrowSchema(schema,
1216 473 : papszOptions);
1217 : }
1218 : #endif
1219 :
1220 : /************************************************************************/
1221 : /* IsArrowSchemaSupported() */
1222 : /************************************************************************/
1223 :
1224 : #if PARQUET_VERSION_MAJOR > 10
1225 1077 : bool OGRParquetWriterLayer::IsArrowSchemaSupported(
1226 : const struct ArrowSchema *schema, CSLConstList papszOptions,
1227 : std::string &osErrorMsg) const
1228 : {
1229 1077 : if (m_poTmpGPKGLayer)
1230 : {
1231 : // When using SORT_BY_BBOX=YES option, we can't directly write the
1232 : // input array, because we need to sort features. But this process
1233 : // only supports the base Arrow types supported by
1234 : // OGRLayer::WriteArrowBatch()
1235 0 : return OGRLayer::IsArrowSchemaSupported(schema, papszOptions,
1236 0 : osErrorMsg);
1237 : }
1238 :
1239 1077 : if (schema->format[0] == 'e' && schema->format[1] == 0)
1240 : {
1241 1 : osErrorMsg = "float16 not supported";
1242 1 : return false;
1243 : }
1244 1076 : if (schema->format[0] == 'v' && schema->format[1] == 'u')
1245 : {
1246 1 : osErrorMsg = "StringView not supported";
1247 1 : return false;
1248 : }
1249 1075 : if (schema->format[0] == 'v' && schema->format[1] == 'z')
1250 : {
1251 1 : osErrorMsg = "BinaryView not supported";
1252 1 : return false;
1253 : }
1254 1074 : if (schema->format[0] == '+' && schema->format[1] == 'v')
1255 : {
1256 0 : if (schema->format[2] == 'l')
1257 : {
1258 0 : osErrorMsg = "ListView not supported";
1259 0 : return false;
1260 : }
1261 0 : else if (schema->format[2] == 'L')
1262 : {
1263 0 : osErrorMsg = "LargeListView not supported";
1264 0 : return false;
1265 : }
1266 : }
1267 2136 : for (int64_t i = 0; i < schema->n_children; ++i)
1268 : {
1269 1065 : if (!IsArrowSchemaSupported(schema->children[i], papszOptions,
1270 : osErrorMsg))
1271 : {
1272 3 : return false;
1273 : }
1274 : }
1275 1071 : return true;
1276 : }
1277 : #endif
1278 :
1279 : /************************************************************************/
1280 : /* SetMetadata() */
1281 : /************************************************************************/
1282 :
1283 13 : CPLErr OGRParquetWriterLayer::SetMetadata(char **papszMetadata,
1284 : const char *pszDomain)
1285 : {
1286 13 : if (!pszDomain || !EQUAL(pszDomain, "SHAPEFILE"))
1287 : {
1288 8 : return OGRLayer::SetMetadata(papszMetadata, pszDomain);
1289 : }
1290 5 : return CE_None;
1291 : }
1292 :
1293 : /************************************************************************/
1294 : /* GetDataset() */
1295 : /************************************************************************/
1296 :
1297 23 : GDALDataset *OGRParquetWriterLayer::GetDataset()
1298 : {
1299 23 : return m_poDataset;
1300 : }
|
