Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: Parquet Translator
4 : * Purpose: Implements OGRParquetDriver.
5 : * Author: Even Rouault, <even.rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2022, Planet Labs
9 : *
10 : * SPDX-License-Identifier: MIT
11 : ****************************************************************************/
12 :
13 : #ifdef STANDALONE
14 : #include "gdal_version.h"
15 : #else
16 : #undef DO_NOT_DEFINE_GDAL_DATE_NAME
17 : #include "gdal_version_full/gdal_version.h"
18 : #endif
19 :
20 : #include "ogr_parquet.h"
21 :
22 : #include "../arrow_common/ograrrowwriterlayer.hpp"
23 :
24 : #include "ogr_wkb.h"
25 :
26 : #include <cassert>
27 : #include <utility>
28 :
29 : /************************************************************************/
30 : /* OGRParquetWriterLayer() */
31 : /************************************************************************/
32 :
33 272 : OGRParquetWriterLayer::OGRParquetWriterLayer(
34 : OGRParquetWriterDataset *poDataset, arrow::MemoryPool *poMemoryPool,
35 : const std::shared_ptr<arrow::io::OutputStream> &poOutputStream,
36 272 : const char *pszLayerName)
37 : : OGRArrowWriterLayer(poMemoryPool, poOutputStream, pszLayerName),
38 272 : m_poDataset(poDataset)
39 : {
40 272 : m_bWriteFieldArrowExtensionName = CPLTestBool(
41 : CPLGetConfigOption("OGR_PARQUET_WRITE_ARROW_EXTENSION_NAME", "NO"));
42 272 : }
43 :
44 : /************************************************************************/
45 : /* Close() */
46 : /************************************************************************/
47 :
48 269 : bool OGRParquetWriterLayer::Close()
49 : {
50 269 : if (m_poTmpGPKGLayer)
51 : {
52 2 : if (!CopyTmpGpkgLayerToFinalFile())
53 0 : return false;
54 : }
55 :
56 269 : if (m_bInitializationOK)
57 : {
58 269 : if (!FinalizeWriting())
59 0 : return false;
60 : }
61 :
62 269 : return true;
63 : }
64 :
65 : /************************************************************************/
66 : /* CopyTmpGpkgLayerToFinalFile() */
67 : /************************************************************************/
68 :
69 2 : bool OGRParquetWriterLayer::CopyTmpGpkgLayerToFinalFile()
70 : {
71 2 : if (!m_poTmpGPKGLayer)
72 : {
73 0 : return true;
74 : }
75 :
76 2 : CPLDebug("PARQUET", "CopyTmpGpkgLayerToFinalFile(): start...");
77 :
78 2 : VSIUnlink(m_poTmpGPKG->GetDescription());
79 :
80 4 : OGRFeature oFeat(m_poFeatureDefn);
81 :
82 : // Interval in terms of features between 2 debug progress report messages
83 2 : constexpr int PROGRESS_FC_INTERVAL = 100 * 1000;
84 :
85 : // First, write features without geometries
86 : {
87 2 : auto poTmpLayer = std::unique_ptr<OGRLayer>(m_poTmpGPKG->ExecuteSQL(
88 : "SELECT serialized_feature FROM tmp WHERE fid NOT IN (SELECT id "
89 : "FROM rtree_tmp_geom)",
90 2 : nullptr, nullptr));
91 2 : if (!poTmpLayer)
92 0 : return false;
93 1004 : for (const auto &poSrcFeature : poTmpLayer.get())
94 : {
95 1002 : int nBytesFeature = 0;
96 : const GByte *pabyFeatureData =
97 1002 : poSrcFeature->GetFieldAsBinary(0, &nBytesFeature);
98 1002 : if (!oFeat.DeserializeFromBinary(pabyFeatureData, nBytesFeature))
99 : {
100 0 : CPLError(CE_Failure, CPLE_AppDefined,
101 : "Cannot deserialize feature");
102 0 : return false;
103 : }
104 1002 : if (OGRArrowWriterLayer::ICreateFeature(&oFeat) != OGRERR_NONE)
105 : {
106 0 : return false;
107 : }
108 :
109 1002 : if ((m_nFeatureCount % PROGRESS_FC_INTERVAL) == 0)
110 : {
111 0 : CPLDebugProgress(
112 : "PARQUET",
113 : "CopyTmpGpkgLayerToFinalFile(): %.02f%% progress",
114 0 : 100.0 * double(m_nFeatureCount) /
115 0 : double(m_nTmpFeatureCount));
116 : }
117 : }
118 :
119 2 : if (!FlushFeatures())
120 : {
121 0 : return false;
122 : }
123 : }
124 :
125 : // Now walk through the GPKG RTree for features with geometries
126 : // Cf https://github.com/sqlite/sqlite/blob/master/ext/rtree/rtree.c
127 : // for the description of the content of the rtree _node table
128 4 : std::vector<std::pair<int64_t, int>> aNodeNoDepthPair;
129 2 : int nTreeDepth = 0;
130 : // Queue the root node
131 : aNodeNoDepthPair.emplace_back(
132 2 : std::make_pair(/* nodeNo = */ 1, /* depth = */ 0));
133 2 : int nCountWrittenFeaturesSinceLastFlush = 0;
134 50 : while (!aNodeNoDepthPair.empty())
135 : {
136 48 : const auto &oLastPair = aNodeNoDepthPair.back();
137 48 : const int64_t nNodeNo = oLastPair.first;
138 48 : const int nCurDepth = oLastPair.second;
139 : //CPLDebug("PARQUET", "Reading nodeNode=%d, curDepth=%d", int(nNodeNo), nCurDepth);
140 48 : aNodeNoDepthPair.pop_back();
141 :
142 48 : auto poRTreeLayer = std::unique_ptr<OGRLayer>(m_poTmpGPKG->ExecuteSQL(
143 : CPLSPrintf("SELECT data FROM rtree_tmp_geom_node WHERE nodeno "
144 : "= " CPL_FRMT_GIB,
145 : static_cast<GIntBig>(nNodeNo)),
146 48 : nullptr, nullptr));
147 48 : if (!poRTreeLayer)
148 : {
149 0 : CPLError(CE_Failure, CPLE_AppDefined,
150 : "Cannot read node " CPL_FRMT_GIB,
151 : static_cast<GIntBig>(nNodeNo));
152 0 : return false;
153 : }
154 : const auto poRTreeFeature =
155 48 : std::unique_ptr<const OGRFeature>(poRTreeLayer->GetNextFeature());
156 48 : if (!poRTreeFeature)
157 : {
158 0 : CPLError(CE_Failure, CPLE_AppDefined,
159 : "Cannot read node " CPL_FRMT_GIB,
160 : static_cast<GIntBig>(nNodeNo));
161 0 : return false;
162 : }
163 :
164 48 : int nNodeBytes = 0;
165 : const GByte *pabyNodeData =
166 48 : poRTreeFeature->GetFieldAsBinary(0, &nNodeBytes);
167 48 : constexpr int BLOB_HEADER_SIZE = 4;
168 48 : if (nNodeBytes < BLOB_HEADER_SIZE)
169 : {
170 0 : CPLError(CE_Failure, CPLE_AppDefined,
171 : "Not enough bytes when reading node " CPL_FRMT_GIB,
172 : static_cast<GIntBig>(nNodeNo));
173 0 : return false;
174 : }
175 48 : if (nNodeNo == 1)
176 : {
177 : // Get the RTree depth from the root node
178 2 : nTreeDepth = (pabyNodeData[0] << 8) | pabyNodeData[1];
179 : //CPLDebug("PARQUET", "nTreeDepth = %d", nTreeDepth);
180 : }
181 :
182 48 : const int nCellCount = (pabyNodeData[2] << 8) | pabyNodeData[3];
183 48 : constexpr int SIZEOF_CELL = 24; // int64_t + 4 float
184 48 : if (nNodeBytes < BLOB_HEADER_SIZE + SIZEOF_CELL * nCellCount)
185 : {
186 0 : CPLError(CE_Failure, CPLE_AppDefined,
187 : "Not enough bytes when reading node " CPL_FRMT_GIB,
188 : static_cast<GIntBig>(nNodeNo));
189 0 : return false;
190 : }
191 :
192 48 : size_t nOffset = BLOB_HEADER_SIZE;
193 48 : if (nCurDepth == nTreeDepth)
194 : {
195 : // Leaf node: it references feature IDs.
196 :
197 : // If we are about to go above m_nRowGroupSize, flush past
198 : // features now, to improve the spatial compacity of the row group.
199 46 : if (m_nRowGroupSize > nCellCount &&
200 46 : nCountWrittenFeaturesSinceLastFlush + nCellCount >
201 46 : m_nRowGroupSize)
202 : {
203 14 : nCountWrittenFeaturesSinceLastFlush = 0;
204 14 : if (!FlushFeatures())
205 : {
206 0 : return false;
207 : }
208 : }
209 :
210 : // nCellCount shouldn't be over 51 normally, but even 65535
211 : // would be fine...
212 46 : assert(nCellCount <= 65535);
213 1248 : for (int i = 0; i < nCellCount; ++i)
214 : {
215 : int64_t nFID;
216 1202 : memcpy(&nFID, pabyNodeData + nOffset, sizeof(int64_t));
217 1202 : CPL_MSBPTR64(&nFID);
218 :
219 : const auto poSrcFeature = std::unique_ptr<const OGRFeature>(
220 1202 : m_poTmpGPKGLayer->GetFeature(nFID));
221 1202 : if (!poSrcFeature)
222 : {
223 0 : CPLError(CE_Failure, CPLE_AppDefined,
224 : "Cannot get feature " CPL_FRMT_GIB,
225 : static_cast<GIntBig>(nFID));
226 0 : return false;
227 : }
228 :
229 1202 : int nBytesFeature = 0;
230 : const GByte *pabyFeatureData =
231 1202 : poSrcFeature->GetFieldAsBinary(0, &nBytesFeature);
232 1202 : if (!oFeat.DeserializeFromBinary(pabyFeatureData,
233 : nBytesFeature))
234 : {
235 0 : CPLError(CE_Failure, CPLE_AppDefined,
236 : "Cannot deserialize feature");
237 0 : return false;
238 : }
239 1202 : if (OGRArrowWriterLayer::ICreateFeature(&oFeat) != OGRERR_NONE)
240 : {
241 0 : return false;
242 : }
243 :
244 1202 : nOffset += SIZEOF_CELL;
245 :
246 1202 : ++nCountWrittenFeaturesSinceLastFlush;
247 :
248 1202 : if ((m_nFeatureCount % PROGRESS_FC_INTERVAL) == 0 ||
249 1202 : m_nFeatureCount == m_nTmpFeatureCount / 2)
250 : {
251 2 : CPLDebugProgress(
252 : "PARQUET",
253 : "CopyTmpGpkgLayerToFinalFile(): %.02f%% progress",
254 2 : 100.0 * double(m_nFeatureCount) /
255 2 : double(m_nTmpFeatureCount));
256 : }
257 : }
258 : }
259 : else
260 : {
261 : // Non-leaf node: it references child nodes.
262 :
263 : // nCellCount shouldn't be over 51 normally, but even 65535
264 : // would be fine...
265 2 : assert(nCellCount <= 65535);
266 48 : for (int i = 0; i < nCellCount; ++i)
267 : {
268 : int64_t nNode;
269 46 : memcpy(&nNode, pabyNodeData + nOffset, sizeof(int64_t));
270 46 : CPL_MSBPTR64(&nNode);
271 : aNodeNoDepthPair.emplace_back(
272 46 : std::make_pair(nNode, nCurDepth + 1));
273 46 : nOffset += SIZEOF_CELL;
274 : }
275 : }
276 : }
277 :
278 2 : CPLDebug("PARQUET",
279 : "CopyTmpGpkgLayerToFinalFile(): 100%%, successfully finished");
280 2 : return true;
281 : }
282 :
283 : /************************************************************************/
284 : /* IsSupportedGeometryType() */
285 : /************************************************************************/
286 :
287 271 : bool OGRParquetWriterLayer::IsSupportedGeometryType(
288 : OGRwkbGeometryType eGType) const
289 : {
290 271 : const auto eFlattenType = wkbFlatten(eGType);
291 271 : if (!OGR_GT_HasM(eGType) && eFlattenType <= wkbGeometryCollection)
292 : {
293 270 : return true;
294 : }
295 :
296 : const auto osConfigOptionName =
297 3 : "OGR_" + GetDriverUCName() + "_ALLOW_ALL_DIMS";
298 1 : if (CPLTestBool(CPLGetConfigOption(osConfigOptionName.c_str(), "NO")))
299 : {
300 0 : return true;
301 : }
302 :
303 1 : CPLError(CE_Failure, CPLE_NotSupported,
304 : "Only 2D and Z geometry types are supported (unless the "
305 : "%s configuration option is set to YES)",
306 : osConfigOptionName.c_str());
307 1 : return false;
308 : }
309 :
310 : /************************************************************************/
311 : /* SetOptions() */
312 : /************************************************************************/
313 :
314 272 : bool OGRParquetWriterLayer::SetOptions(CSLConstList papszOptions,
315 : const OGRSpatialReference *poSpatialRef,
316 : OGRwkbGeometryType eGType)
317 : {
318 272 : const char *pszWriteCoveringBBox = CSLFetchNameValueDef(
319 : papszOptions, "WRITE_COVERING_BBOX",
320 : CPLGetConfigOption("OGR_PARQUET_WRITE_COVERING_BBOX", nullptr));
321 272 : m_bWriteBBoxStruct =
322 272 : pszWriteCoveringBBox == nullptr || CPLTestBool(pszWriteCoveringBBox);
323 :
324 272 : if (CPLTestBool(CSLFetchNameValueDef(papszOptions, "SORT_BY_BBOX", "NO")))
325 : {
326 6 : const std::string osTmpGPKG(std::string(m_poDataset->GetDescription()) +
327 3 : ".tmp.gpkg");
328 3 : auto poGPKGDrv = GetGDALDriverManager()->GetDriverByName("GPKG");
329 3 : if (!poGPKGDrv)
330 : {
331 1 : CPLError(
332 : CE_Failure, CPLE_AppDefined,
333 : "Driver GPKG required for SORT_BY_BBOX layer creation option");
334 1 : return false;
335 : }
336 2 : m_poTmpGPKG.reset(poGPKGDrv->Create(osTmpGPKG.c_str(), 0, 0, 0,
337 : GDT_Unknown, nullptr));
338 2 : if (!m_poTmpGPKG)
339 0 : return false;
340 2 : m_poTmpGPKG->MarkSuppressOnClose();
341 2 : m_poTmpGPKGLayer = m_poTmpGPKG->CreateLayer("tmp");
342 2 : if (!m_poTmpGPKGLayer)
343 0 : return false;
344 : // Serialized feature
345 2 : m_poTmpGPKGLayer->CreateField(
346 2 : std::make_unique<OGRFieldDefn>("serialized_feature", OFTBinary)
347 2 : .get());
348 2 : CPL_IGNORE_RET_VAL(m_poTmpGPKGLayer->StartTransaction());
349 : }
350 :
351 : const char *pszGeomEncoding =
352 271 : CSLFetchNameValue(papszOptions, "GEOMETRY_ENCODING");
353 271 : m_eGeomEncoding = OGRArrowGeomEncoding::WKB;
354 271 : if (pszGeomEncoding)
355 : {
356 148 : if (EQUAL(pszGeomEncoding, "WKB"))
357 0 : m_eGeomEncoding = OGRArrowGeomEncoding::WKB;
358 148 : else if (EQUAL(pszGeomEncoding, "WKT"))
359 8 : m_eGeomEncoding = OGRArrowGeomEncoding::WKT;
360 140 : else if (EQUAL(pszGeomEncoding, "GEOARROW_INTERLEAVED"))
361 : {
362 28 : CPLErrorOnce(
363 : CE_Warning, CPLE_AppDefined,
364 : "Use of GEOMETRY_ENCODING=GEOARROW_INTERLEAVED is not "
365 : "recommended. "
366 : "GeoParquet 1.1 uses GEOMETRY_ENCODING=GEOARROW (struct) "
367 : "instead.");
368 28 : m_eGeomEncoding = OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC;
369 : }
370 112 : else if (EQUAL(pszGeomEncoding, "GEOARROW") ||
371 0 : EQUAL(pszGeomEncoding, "GEOARROW_STRUCT"))
372 112 : m_eGeomEncoding = OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC;
373 : else
374 : {
375 0 : CPLError(CE_Failure, CPLE_NotSupported,
376 : "Unsupported GEOMETRY_ENCODING = %s", pszGeomEncoding);
377 0 : return false;
378 : }
379 : }
380 :
381 : const char *pszCoordPrecision =
382 271 : CSLFetchNameValue(papszOptions, "COORDINATE_PRECISION");
383 271 : if (pszCoordPrecision)
384 0 : m_nWKTCoordinatePrecision = atoi(pszCoordPrecision);
385 :
386 271 : m_bForceCounterClockwiseOrientation =
387 271 : EQUAL(CSLFetchNameValueDef(papszOptions, "POLYGON_ORIENTATION",
388 : "COUNTERCLOCKWISE"),
389 : "COUNTERCLOCKWISE");
390 :
391 271 : if (eGType != wkbNone)
392 : {
393 245 : if (!IsSupportedGeometryType(eGType))
394 : {
395 1 : return false;
396 : }
397 :
398 244 : m_poFeatureDefn->SetGeomType(eGType);
399 244 : auto eGeomEncoding = m_eGeomEncoding;
400 244 : if (eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC ||
401 216 : eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC)
402 : {
403 140 : const auto eEncodingType = eGeomEncoding;
404 140 : eGeomEncoding = GetPreciseArrowGeomEncoding(eEncodingType, eGType);
405 140 : if (eGeomEncoding == eEncodingType)
406 0 : return false;
407 : }
408 244 : m_aeGeomEncoding.push_back(eGeomEncoding);
409 244 : m_poFeatureDefn->GetGeomFieldDefn(0)->SetName(
410 : CSLFetchNameValueDef(papszOptions, "GEOMETRY_NAME", "geometry"));
411 244 : if (poSpatialRef)
412 : {
413 27 : auto poSRS = poSpatialRef->Clone();
414 27 : m_poFeatureDefn->GetGeomFieldDefn(0)->SetSpatialRef(poSRS);
415 27 : poSRS->Release();
416 : }
417 : }
418 :
419 270 : m_osFIDColumn = CSLFetchNameValueDef(papszOptions, "FID", "");
420 :
421 270 : const char *pszCompression = CSLFetchNameValue(papszOptions, "COMPRESSION");
422 270 : if (pszCompression == nullptr)
423 : {
424 792 : auto oResult = arrow::util::Codec::GetCompressionType("snappy");
425 264 : if (oResult.ok() && arrow::util::Codec::IsAvailable(*oResult))
426 : {
427 264 : pszCompression = "SNAPPY";
428 : }
429 : else
430 : {
431 0 : pszCompression = "NONE";
432 : }
433 : }
434 :
435 270 : if (EQUAL(pszCompression, "NONE"))
436 0 : pszCompression = "UNCOMPRESSED";
437 : auto oResult = arrow::util::Codec::GetCompressionType(
438 540 : CPLString(pszCompression).tolower());
439 270 : if (!oResult.ok())
440 : {
441 1 : CPLError(CE_Failure, CPLE_NotSupported,
442 : "Unrecognized compression method: %s", pszCompression);
443 1 : return false;
444 : }
445 269 : m_eCompression = *oResult;
446 269 : if (!arrow::util::Codec::IsAvailable(m_eCompression))
447 : {
448 0 : CPLError(CE_Failure, CPLE_NotSupported,
449 : "Compression method %s is known, but libarrow has not "
450 : "been built with support for it",
451 : pszCompression);
452 0 : return false;
453 : }
454 269 : m_oWriterPropertiesBuilder.compression(m_eCompression);
455 :
456 : const char *pszCompressionLevel =
457 269 : CSLFetchNameValue(papszOptions, "COMPRESSION_LEVEL");
458 269 : if (pszCompressionLevel)
459 : {
460 2 : const int nCompressionLevel = atoi(pszCompressionLevel);
461 2 : if (nCompressionLevel != DEFAULT_COMPRESSION_LEVEL)
462 2 : m_oWriterPropertiesBuilder.compression_level(nCompressionLevel);
463 : }
464 :
465 : const std::string osCreator =
466 269 : CSLFetchNameValueDef(papszOptions, "CREATOR", "");
467 269 : if (!osCreator.empty())
468 1 : m_oWriterPropertiesBuilder.created_by(osCreator);
469 : else
470 268 : m_oWriterPropertiesBuilder.created_by("GDAL " GDAL_RELEASE_NAME
471 : ", using " CREATED_BY_VERSION);
472 :
473 : // Undocumented option. Not clear it is useful besides unit test purposes
474 269 : if (!CPLTestBool(CSLFetchNameValueDef(papszOptions, "STATISTICS", "YES")))
475 1 : m_oWriterPropertiesBuilder.disable_statistics();
476 :
477 : #if PARQUET_VERSION_MAJOR >= 12
478 : // Undocumented option. Not clear it is useful to disable it.
479 269 : if (CPLTestBool(CSLFetchNameValueDef(papszOptions, "PAGE_INDEX", "YES")))
480 269 : m_oWriterPropertiesBuilder.enable_write_page_index();
481 : #endif
482 :
483 : const char *pszWriteGeo =
484 269 : CPLGetConfigOption("OGR_PARQUET_WRITE_GEO", nullptr);
485 269 : m_bWriteGeoMetadata = pszWriteGeo == nullptr || CPLTestBool(pszWriteGeo);
486 :
487 269 : if (m_eGeomEncoding == OGRArrowGeomEncoding::WKB && eGType != wkbNone)
488 : {
489 : #if ARROW_VERSION_MAJOR >= 21
490 : const char *pszUseParquetGeoTypes =
491 : CSLFetchNameValueDef(papszOptions, "USE_PARQUET_GEO_TYPES", "NO");
492 : if (EQUAL(pszUseParquetGeoTypes, "ONLY"))
493 : {
494 : m_bUseArrowWKBExtension = true;
495 : if (pszWriteGeo == nullptr)
496 : m_bWriteGeoMetadata = false;
497 : if (pszWriteCoveringBBox == nullptr)
498 : m_bWriteBBoxStruct = false;
499 : }
500 : else
501 : {
502 : m_bUseArrowWKBExtension = CPLTestBool(pszUseParquetGeoTypes);
503 : }
504 : #else
505 103 : m_oWriterPropertiesBuilder.disable_statistics(
506 309 : parquet::schema::ColumnPath::FromDotString(
507 103 : m_poFeatureDefn->GetGeomFieldDefn(0)->GetNameRef()));
508 : #endif
509 : }
510 :
511 : const char *pszRowGroupSize =
512 269 : CSLFetchNameValue(papszOptions, "ROW_GROUP_SIZE");
513 269 : if (pszRowGroupSize)
514 : {
515 10 : auto nRowGroupSize = static_cast<int64_t>(atoll(pszRowGroupSize));
516 10 : if (nRowGroupSize > 0)
517 : {
518 10 : if (nRowGroupSize > INT_MAX)
519 0 : nRowGroupSize = INT_MAX;
520 10 : m_nRowGroupSize = nRowGroupSize;
521 : }
522 : }
523 :
524 269 : m_bEdgesSpherical = EQUAL(
525 : CSLFetchNameValueDef(papszOptions, "EDGES", "PLANAR"), "SPHERICAL");
526 :
527 269 : m_bInitializationOK = true;
528 269 : return true;
529 : }
530 :
531 : /************************************************************************/
532 : /* CloseFileWriter() */
533 : /************************************************************************/
534 :
535 269 : bool OGRParquetWriterLayer::CloseFileWriter()
536 : {
537 538 : auto status = m_poFileWriter->Close();
538 269 : if (!status.ok())
539 : {
540 0 : CPLError(CE_Failure, CPLE_AppDefined,
541 : "FileWriter::Close() failed with %s",
542 0 : status.message().c_str());
543 : }
544 538 : return status.ok();
545 : }
546 :
547 : /************************************************************************/
548 : /* GetGeoMetadata() */
549 : /************************************************************************/
550 :
551 269 : std::string OGRParquetWriterLayer::GetGeoMetadata() const
552 : {
553 : // Just for unit testing purposes
554 : const char *pszGeoMetadata =
555 269 : CPLGetConfigOption("OGR_PARQUET_GEO_METADATA", nullptr);
556 269 : if (pszGeoMetadata)
557 16 : return pszGeoMetadata;
558 :
559 253 : if (m_poFeatureDefn->GetGeomFieldCount() != 0 && m_bWriteGeoMetadata)
560 : {
561 472 : CPLJSONObject oRoot;
562 236 : oRoot.Add("version", "1.1.0");
563 236 : oRoot.Add("primary_column",
564 236 : m_poFeatureDefn->GetGeomFieldDefn(0)->GetNameRef());
565 472 : CPLJSONObject oColumns;
566 236 : oRoot.Add("columns", oColumns);
567 489 : for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
568 : {
569 253 : const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
570 506 : CPLJSONObject oColumn;
571 253 : oColumns.Add(poGeomFieldDefn->GetNameRef(), oColumn);
572 253 : oColumn.Add("encoding",
573 253 : GetGeomEncodingAsString(m_aeGeomEncoding[i], true));
574 :
575 253 : if (CPLTestBool(CPLGetConfigOption("OGR_PARQUET_WRITE_CRS", "YES")))
576 : {
577 252 : const auto poSRS = poGeomFieldDefn->GetSpatialRef();
578 252 : if (poSRS)
579 : {
580 52 : OGRSpatialReference oSRSIdentified(IdentifyCRS(poSRS));
581 :
582 : const char *pszAuthName =
583 26 : oSRSIdentified.GetAuthorityName(nullptr);
584 : const char *pszAuthCode =
585 26 : oSRSIdentified.GetAuthorityCode(nullptr);
586 :
587 26 : bool bOmitCRS = false;
588 26 : if (pszAuthName != nullptr && pszAuthCode != nullptr &&
589 25 : ((EQUAL(pszAuthName, "EPSG") &&
590 22 : EQUAL(pszAuthCode, "4326")) ||
591 14 : (EQUAL(pszAuthName, "OGC") &&
592 3 : EQUAL(pszAuthCode, "CRS84"))))
593 : {
594 : // To make things less confusing for non-geo-aware
595 : // consumers, omit EPSG:4326 / OGC:CRS84 CRS by default
596 14 : bOmitCRS = CPLTestBool(CPLGetConfigOption(
597 : "OGR_PARQUET_CRS_OMIT_IF_WGS84", "YES"));
598 : }
599 :
600 26 : if (bOmitCRS)
601 : {
602 : // do nothing
603 : }
604 12 : else if (EQUAL(CPLGetConfigOption(
605 : "OGR_PARQUET_CRS_ENCODING", "PROJJSON"),
606 : "PROJJSON"))
607 : {
608 : // CRS encoded as PROJJSON for GeoParquet >= 0.4.0
609 12 : char *pszPROJJSON = nullptr;
610 12 : oSRSIdentified.exportToPROJJSON(&pszPROJJSON, nullptr);
611 24 : CPLJSONDocument oCRSDoc;
612 12 : CPL_IGNORE_RET_VAL(oCRSDoc.LoadMemory(pszPROJJSON));
613 12 : CPLFree(pszPROJJSON);
614 12 : CPLJSONObject oCRSRoot = oCRSDoc.GetRoot();
615 12 : RemoveIDFromMemberOfEnsembles(oCRSRoot);
616 12 : oColumn.Add("crs", oCRSRoot);
617 : }
618 : else
619 : {
620 : // WKT was used in GeoParquet <= 0.3.0
621 0 : const char *const apszOptions[] = {
622 : "FORMAT=WKT2_2019", "MULTILINE=NO", nullptr};
623 0 : char *pszWKT = nullptr;
624 0 : oSRSIdentified.exportToWkt(&pszWKT, apszOptions);
625 0 : if (pszWKT)
626 0 : oColumn.Add("crs", pszWKT);
627 0 : CPLFree(pszWKT);
628 : }
629 :
630 26 : const double dfCoordEpoch = poSRS->GetCoordinateEpoch();
631 26 : if (dfCoordEpoch > 0)
632 2 : oColumn.Add("epoch", dfCoordEpoch);
633 : }
634 : else
635 : {
636 226 : oColumn.AddNull("crs");
637 : }
638 : }
639 :
640 253 : if (m_bEdgesSpherical)
641 : {
642 1 : oColumn.Add("edges", "spherical");
643 : }
644 :
645 479 : if (m_aoEnvelopes[i].IsInit() &&
646 226 : CPLTestBool(
647 : CPLGetConfigOption("OGR_PARQUET_WRITE_BBOX", "YES")))
648 : {
649 226 : bool bHasZ = false;
650 411 : for (const auto eGeomType : m_oSetWrittenGeometryTypes[i])
651 : {
652 268 : bHasZ = OGR_GT_HasZ(eGeomType);
653 268 : if (bHasZ)
654 83 : break;
655 : }
656 226 : CPLJSONArray oBBOX;
657 226 : oBBOX.Add(m_aoEnvelopes[i].MinX);
658 226 : oBBOX.Add(m_aoEnvelopes[i].MinY);
659 226 : if (bHasZ)
660 83 : oBBOX.Add(m_aoEnvelopes[i].MinZ);
661 226 : oBBOX.Add(m_aoEnvelopes[i].MaxX);
662 226 : oBBOX.Add(m_aoEnvelopes[i].MaxY);
663 226 : if (bHasZ)
664 83 : oBBOX.Add(m_aoEnvelopes[i].MaxZ);
665 226 : oColumn.Add("bbox", oBBOX);
666 : }
667 :
668 : // Bounding box column definition
669 436 : if (m_bWriteBBoxStruct &&
670 183 : CPLTestBool(CPLGetConfigOption(
671 : "OGR_PARQUET_WRITE_COVERING_BBOX_IN_METADATA", "YES")))
672 : {
673 366 : CPLJSONObject oCovering;
674 183 : oColumn.Add("covering", oCovering);
675 366 : CPLJSONObject oBBOX;
676 183 : oCovering.Add("bbox", oBBOX);
677 : const auto AddComponent =
678 2196 : [this, i, &oBBOX](const char *pszComponent)
679 : {
680 732 : CPLJSONArray oArray;
681 732 : oArray.Add(m_apoFieldsBBOX[i]->name());
682 732 : oArray.Add(pszComponent);
683 732 : oBBOX.Add(pszComponent, oArray);
684 732 : };
685 183 : AddComponent("xmin");
686 183 : AddComponent("ymin");
687 183 : AddComponent("xmax");
688 183 : AddComponent("ymax");
689 : }
690 :
691 284 : const auto GetStringGeometryType = [](OGRwkbGeometryType eType)
692 : {
693 284 : const auto eFlattenType = wkbFlatten(eType);
694 284 : std::string osType = "Unknown";
695 284 : if (wkbPoint == eFlattenType)
696 66 : osType = "Point";
697 218 : else if (wkbLineString == eFlattenType)
698 34 : osType = "LineString";
699 184 : else if (wkbPolygon == eFlattenType)
700 55 : osType = "Polygon";
701 129 : else if (wkbMultiPoint == eFlattenType)
702 26 : osType = "MultiPoint";
703 103 : else if (wkbMultiLineString == eFlattenType)
704 29 : osType = "MultiLineString";
705 74 : else if (wkbMultiPolygon == eFlattenType)
706 69 : osType = "MultiPolygon";
707 5 : else if (wkbGeometryCollection == eFlattenType)
708 5 : osType = "GeometryCollection";
709 284 : if (osType != "Unknown")
710 : {
711 : // M and ZM not supported officially currently, but it
712 : // doesn't hurt to anticipate
713 284 : if (OGR_GT_HasZ(eType) && OGR_GT_HasM(eType))
714 8 : osType += " ZM";
715 276 : else if (OGR_GT_HasZ(eType))
716 91 : osType += " Z";
717 185 : else if (OGR_GT_HasM(eType))
718 8 : osType += " M";
719 : }
720 284 : return osType;
721 : };
722 :
723 253 : if (m_bForceCounterClockwiseOrientation)
724 252 : oColumn.Add("orientation", "counterclockwise");
725 :
726 253 : CPLJSONArray oArray;
727 537 : for (const auto eType : m_oSetWrittenGeometryTypes[i])
728 : {
729 284 : oArray.Add(GetStringGeometryType(eType));
730 : }
731 253 : oColumn.Add("geometry_types", oArray);
732 : }
733 :
734 236 : return oRoot.Format(CPLJSONObject::PrettyFormat::Plain);
735 : }
736 17 : return std::string();
737 : }
738 :
739 : /************************************************************************/
740 : /* PerformStepsBeforeFinalFlushGroup() */
741 : /************************************************************************/
742 :
743 269 : void OGRParquetWriterLayer::PerformStepsBeforeFinalFlushGroup()
744 : {
745 269 : if (m_poKeyValueMetadata)
746 : {
747 538 : std::string osGeoMetadata = GetGeoMetadata();
748 538 : auto poTmpSchema = m_poSchema;
749 269 : if (!osGeoMetadata.empty())
750 : {
751 : // HACK: it would be good for Arrow to provide a clean way to alter
752 : // key value metadata before finalizing.
753 : // We need to write metadata at end to write the bounding box.
754 252 : const_cast<arrow::KeyValueMetadata *>(m_poKeyValueMetadata.get())
755 252 : ->Append("geo", osGeoMetadata);
756 :
757 252 : auto kvMetadata = poTmpSchema->metadata()
758 11 : ? poTmpSchema->metadata()->Copy()
759 263 : : std::make_shared<arrow::KeyValueMetadata>();
760 252 : kvMetadata->Append("geo", std::move(osGeoMetadata));
761 252 : poTmpSchema = poTmpSchema->WithMetadata(kvMetadata);
762 : }
763 :
764 269 : if (CPLTestBool(
765 : CPLGetConfigOption("OGR_PARQUET_WRITE_ARROW_SCHEMA", "YES")))
766 : {
767 : auto status =
768 538 : ::arrow::ipc::SerializeSchema(*poTmpSchema, m_poMemoryPool);
769 269 : if (status.ok())
770 : {
771 : // The serialized schema is not UTF-8, which is required for
772 : // Thrift
773 538 : const std::string schema_as_string = (*status)->ToString();
774 : std::string schema_base64 =
775 269 : ::arrow::util::base64_encode(schema_as_string);
776 269 : static const std::string kArrowSchemaKey = "ARROW:schema";
777 : const_cast<arrow::KeyValueMetadata *>(
778 269 : m_poKeyValueMetadata.get())
779 269 : ->Append(kArrowSchemaKey, std::move(schema_base64));
780 : }
781 : }
782 :
783 : // Put GDAL metadata into a gdal:metadata domain
784 538 : CPLJSONObject oMultiMetadata;
785 269 : bool bHasMultiMetadata = false;
786 275 : auto &l_oMDMD = oMDMD.GetDomainList() && *(oMDMD.GetDomainList())
787 275 : ? oMDMD
788 263 : : m_poDataset->GetMultiDomainMetadata();
789 277 : for (CSLConstList papszDomainIter = l_oMDMD.GetDomainList();
790 277 : papszDomainIter && *papszDomainIter; ++papszDomainIter)
791 : {
792 8 : const char *pszDomain = *papszDomainIter;
793 8 : CSLConstList papszMD = l_oMDMD.GetMetadata(pszDomain);
794 8 : if (STARTS_WITH(pszDomain, "json:") && papszMD && papszMD[0])
795 : {
796 1 : CPLJSONDocument oDoc;
797 1 : if (oDoc.LoadMemory(papszMD[0]))
798 : {
799 1 : bHasMultiMetadata = true;
800 1 : oMultiMetadata.Add(pszDomain, oDoc.GetRoot());
801 1 : continue;
802 0 : }
803 : }
804 7 : else if (STARTS_WITH(pszDomain, "xml:") && papszMD && papszMD[0])
805 : {
806 1 : bHasMultiMetadata = true;
807 1 : oMultiMetadata.Add(pszDomain, papszMD[0]);
808 1 : continue;
809 : }
810 12 : CPLJSONObject oMetadata;
811 6 : bool bHasMetadata = false;
812 12 : for (CSLConstList papszMDIter = papszMD;
813 12 : papszMDIter && *papszMDIter; ++papszMDIter)
814 : {
815 6 : char *pszKey = nullptr;
816 6 : const char *pszValue = CPLParseNameValue(*papszMDIter, &pszKey);
817 6 : if (pszKey && pszValue)
818 : {
819 6 : bHasMetadata = true;
820 6 : bHasMultiMetadata = true;
821 6 : oMetadata.Add(pszKey, pszValue);
822 : }
823 6 : CPLFree(pszKey);
824 : }
825 6 : if (bHasMetadata)
826 6 : oMultiMetadata.Add(pszDomain, oMetadata);
827 : }
828 269 : if (bHasMultiMetadata)
829 : {
830 6 : const_cast<arrow::KeyValueMetadata *>(m_poKeyValueMetadata.get())
831 6 : ->Append(
832 : "gdal:metadata",
833 12 : oMultiMetadata.Format(CPLJSONObject::PrettyFormat::Plain));
834 : }
835 : }
836 269 : }
837 :
838 : /************************************************************************/
839 : /* Open() */
840 : /************************************************************************/
841 :
842 : // Same as parquet::arrow::FileWriter::Open(), except we also
843 : // return KeyValueMetadata
844 : static arrow::Status
845 269 : Open(const ::arrow::Schema &schema, ::arrow::MemoryPool *pool,
846 : std::shared_ptr<::arrow::io::OutputStream> sink,
847 : std::shared_ptr<parquet::WriterProperties> properties,
848 : std::shared_ptr<parquet::ArrowWriterProperties> arrow_properties,
849 : std::unique_ptr<parquet::arrow::FileWriter> *writer,
850 : std::shared_ptr<const arrow::KeyValueMetadata> *outMetadata)
851 : {
852 269 : std::shared_ptr<parquet::SchemaDescriptor> parquet_schema;
853 538 : RETURN_NOT_OK(parquet::arrow::ToParquetSchema(
854 : &schema, *properties, *arrow_properties, &parquet_schema));
855 :
856 : auto schema_node = std::static_pointer_cast<parquet::schema::GroupNode>(
857 538 : parquet_schema->schema_root());
858 :
859 269 : auto metadata = schema.metadata()
860 16 : ? schema.metadata()->Copy()
861 554 : : std::make_shared<arrow::KeyValueMetadata>();
862 269 : *outMetadata = metadata;
863 :
864 269 : std::unique_ptr<parquet::ParquetFileWriter> base_writer;
865 269 : PARQUET_CATCH_NOT_OK(base_writer = parquet::ParquetFileWriter::Open(
866 : std::move(sink), std::move(schema_node),
867 : std::move(properties), metadata));
868 :
869 269 : auto schema_ptr = std::make_shared<::arrow::Schema>(schema);
870 : return parquet::arrow::FileWriter::Make(
871 538 : pool, std::move(base_writer), std::move(schema_ptr),
872 807 : std::move(arrow_properties), writer);
873 : }
874 :
875 : /************************************************************************/
876 : /* CreateSchema() */
877 : /************************************************************************/
878 :
879 269 : void OGRParquetWriterLayer::CreateSchema()
880 : {
881 269 : CreateSchemaCommon();
882 269 : }
883 :
884 : /************************************************************************/
885 : /* CreateGeomField() */
886 : /************************************************************************/
887 :
888 27 : OGRErr OGRParquetWriterLayer::CreateGeomField(const OGRGeomFieldDefn *poField,
889 : int bApproxOK)
890 : {
891 27 : OGRErr eErr = OGRArrowWriterLayer::CreateGeomField(poField, bApproxOK);
892 53 : if (eErr == OGRERR_NONE &&
893 26 : m_aeGeomEncoding.back() == OGRArrowGeomEncoding::WKB
894 : #if ARROW_VERSION_MAJOR < 21
895 : // Geostatistics in Arrow 21 do not support geographic type for now
896 53 : && m_bEdgesSpherical
897 : #endif
898 : )
899 : {
900 0 : m_oWriterPropertiesBuilder.disable_statistics(
901 0 : parquet::schema::ColumnPath::FromDotString(
902 0 : m_poFeatureDefn
903 0 : ->GetGeomFieldDefn(m_poFeatureDefn->GetGeomFieldCount() - 1)
904 : ->GetNameRef()));
905 : }
906 27 : return eErr;
907 : }
908 :
909 : /************************************************************************/
910 : /* CreateWriter() */
911 : /************************************************************************/
912 :
913 269 : void OGRParquetWriterLayer::CreateWriter()
914 : {
915 269 : CPLAssert(m_poFileWriter == nullptr);
916 :
917 269 : if (m_poSchema == nullptr)
918 : {
919 40 : CreateSchema();
920 : }
921 : else
922 : {
923 229 : FinalizeSchema();
924 : }
925 :
926 : auto arrowWriterProperties =
927 269 : parquet::ArrowWriterProperties::Builder().store_schema()->build();
928 807 : CPL_IGNORE_RET_VAL(Open(*m_poSchema, m_poMemoryPool, m_poOutputStream,
929 538 : m_oWriterPropertiesBuilder.build(),
930 269 : std::move(arrowWriterProperties), &m_poFileWriter,
931 : &m_poKeyValueMetadata));
932 269 : }
933 :
934 : /************************************************************************/
935 : /* ICreateFeature() */
936 : /************************************************************************/
937 :
938 3091 : OGRErr OGRParquetWriterLayer::ICreateFeature(OGRFeature *poFeature)
939 : {
940 : // If not using SORT_BY_BBOX=YES layer creation option, we can directly
941 : // write features to the final Parquet file
942 3091 : if (!m_poTmpGPKGLayer)
943 887 : return OGRArrowWriterLayer::ICreateFeature(poFeature);
944 :
945 : // SORT_BY_BBOX=YES case: we write for now a serialized version of poFeature
946 : // in a temporary GeoPackage file.
947 :
948 2204 : GIntBig nFID = poFeature->GetFID();
949 2204 : if (!m_osFIDColumn.empty() && nFID == OGRNullFID)
950 : {
951 1102 : nFID = m_nTmpFeatureCount;
952 1102 : poFeature->SetFID(nFID);
953 : }
954 2204 : ++m_nTmpFeatureCount;
955 :
956 4408 : std::vector<GByte> abyBuffer;
957 : // Serialize the source feature as a single array of bytes to preserve it
958 : // fully
959 2204 : if (!poFeature->SerializeToBinary(abyBuffer))
960 : {
961 0 : return OGRERR_FAILURE;
962 : }
963 :
964 : // SQLite3 limitation: a row must fit in slightly less than 1 GB.
965 2204 : constexpr int SOME_MARGIN = 128;
966 2204 : if (abyBuffer.size() > 1024 * 1024 * 1024 - SOME_MARGIN)
967 : {
968 0 : CPLError(CE_Failure, CPLE_NotSupported,
969 : "Features larger than 1 GB are not supported");
970 0 : return OGRERR_FAILURE;
971 : }
972 :
973 4408 : OGRFeature oFeat(m_poTmpGPKGLayer->GetLayerDefn());
974 2204 : oFeat.SetFID(nFID);
975 2204 : oFeat.SetField(0, static_cast<int>(abyBuffer.size()), abyBuffer.data());
976 2204 : const auto poSrcGeom = poFeature->GetGeometryRef();
977 2204 : if (poSrcGeom && !poSrcGeom->IsEmpty())
978 : {
979 : // For the purpose of building an RTree, just use the bounding box of
980 : // the geometry as the geometry.
981 1202 : OGREnvelope sEnvelope;
982 1202 : poSrcGeom->getEnvelope(&sEnvelope);
983 2404 : auto poPoly = std::make_unique<OGRPolygon>();
984 2404 : auto poLR = std::make_unique<OGRLinearRing>();
985 1202 : poLR->addPoint(sEnvelope.MinX, sEnvelope.MinY);
986 1202 : poLR->addPoint(sEnvelope.MinX, sEnvelope.MaxY);
987 1202 : poLR->addPoint(sEnvelope.MaxX, sEnvelope.MaxY);
988 1202 : poLR->addPoint(sEnvelope.MaxX, sEnvelope.MinY);
989 1202 : poLR->addPoint(sEnvelope.MinX, sEnvelope.MinY);
990 1202 : poPoly->addRingDirectly(poLR.release());
991 1202 : oFeat.SetGeometryDirectly(poPoly.release());
992 : }
993 2204 : return m_poTmpGPKGLayer->CreateFeature(&oFeat);
994 : }
995 :
996 : /************************************************************************/
997 : /* FlushGroup() */
998 : /************************************************************************/
999 :
1000 253 : bool OGRParquetWriterLayer::FlushGroup()
1001 : {
1002 : #if PARQUET_VERSION_MAJOR >= 20
1003 : auto status = m_poFileWriter->NewRowGroup();
1004 : #else
1005 506 : auto status = m_poFileWriter->NewRowGroup(m_apoBuilders[0]->length());
1006 : #endif
1007 253 : if (!status.ok())
1008 : {
1009 0 : CPLError(CE_Failure, CPLE_AppDefined, "NewRowGroup() failed with %s",
1010 0 : status.message().c_str());
1011 0 : ClearArrayBuilers();
1012 0 : return false;
1013 : }
1014 :
1015 253 : auto ret = WriteArrays(
1016 1025 : [this](const std::shared_ptr<arrow::Field> &field,
1017 1025 : const std::shared_ptr<arrow::Array> &array)
1018 : {
1019 2050 : auto l_status = m_poFileWriter->WriteColumnChunk(*array);
1020 1025 : if (!l_status.ok())
1021 : {
1022 0 : CPLError(CE_Failure, CPLE_AppDefined,
1023 : "WriteColumnChunk() failed for field %s: %s",
1024 0 : field->name().c_str(), l_status.message().c_str());
1025 0 : return false;
1026 : }
1027 1025 : return true;
1028 : });
1029 :
1030 253 : ClearArrayBuilers();
1031 253 : return ret;
1032 : }
1033 :
1034 : /************************************************************************/
1035 : /* FixupWKBGeometryBeforeWriting() */
1036 : /************************************************************************/
1037 :
1038 43 : void OGRParquetWriterLayer::FixupWKBGeometryBeforeWriting(GByte *pabyWkb,
1039 : size_t nLen)
1040 : {
1041 43 : if (!m_bForceCounterClockwiseOrientation)
1042 0 : return;
1043 :
1044 43 : OGRWKBFixupCounterClockWiseExternalRing(pabyWkb, nLen);
1045 : }
1046 :
1047 : /************************************************************************/
1048 : /* FixupGeometryBeforeWriting() */
1049 : /************************************************************************/
1050 :
1051 1354 : void OGRParquetWriterLayer::FixupGeometryBeforeWriting(OGRGeometry *poGeom)
1052 : {
1053 1354 : if (!m_bForceCounterClockwiseOrientation)
1054 3 : return;
1055 :
1056 1351 : const auto eFlattenType = wkbFlatten(poGeom->getGeometryType());
1057 : // Polygon rings MUST follow the right-hand rule for orientation
1058 : // (counterclockwise external rings, clockwise internal rings)
1059 1351 : if (eFlattenType == wkbPolygon)
1060 : {
1061 64 : bool bFirstRing = true;
1062 131 : for (auto poRing : poGeom->toPolygon())
1063 : {
1064 75 : if ((bFirstRing && poRing->isClockwise()) ||
1065 8 : (!bFirstRing && !poRing->isClockwise()))
1066 : {
1067 62 : poRing->reversePoints();
1068 : }
1069 67 : bFirstRing = false;
1070 : }
1071 : }
1072 1287 : else if (eFlattenType == wkbMultiPolygon ||
1073 : eFlattenType == wkbGeometryCollection)
1074 : {
1075 35 : for (auto poSubGeom : poGeom->toGeometryCollection())
1076 : {
1077 21 : FixupGeometryBeforeWriting(poSubGeom);
1078 : }
1079 : }
1080 : }
1081 :
1082 : /************************************************************************/
1083 : /* WriteArrowBatch() */
1084 : /************************************************************************/
1085 :
1086 : #if PARQUET_VERSION_MAJOR > 10
1087 : inline bool
1088 14 : OGRParquetWriterLayer::WriteArrowBatch(const struct ArrowSchema *schema,
1089 : struct ArrowArray *array,
1090 : CSLConstList papszOptions)
1091 : {
1092 14 : if (m_poTmpGPKGLayer)
1093 : {
1094 : // When using SORT_BY_BBOX=YES option, we can't directly write the
1095 : // input array, because we need to sort features. Hence we fallback
1096 : // to the OGRLayer base implementation, which will ultimately call
1097 : // OGRParquetWriterLayer::ICreateFeature()
1098 0 : return OGRLayer::WriteArrowBatch(schema, array, papszOptions);
1099 : }
1100 :
1101 28 : return WriteArrowBatchInternal(
1102 : schema, array, papszOptions,
1103 28 : [this](const std::shared_ptr<arrow::RecordBatch> &poBatch)
1104 : {
1105 28 : auto status = m_poFileWriter->NewBufferedRowGroup();
1106 14 : if (!status.ok())
1107 : {
1108 0 : CPLError(CE_Failure, CPLE_AppDefined,
1109 : "NewBufferedRowGroup() failed with %s",
1110 0 : status.message().c_str());
1111 0 : return false;
1112 : }
1113 :
1114 14 : status = m_poFileWriter->WriteRecordBatch(*poBatch);
1115 14 : if (!status.ok())
1116 : {
1117 0 : CPLError(CE_Failure, CPLE_AppDefined,
1118 : "WriteRecordBatch() failed: %s",
1119 0 : status.message().c_str());
1120 0 : return false;
1121 : }
1122 :
1123 14 : return true;
1124 14 : });
1125 : }
1126 : #endif
1127 :
1128 : /************************************************************************/
1129 : /* TestCapability() */
1130 : /************************************************************************/
1131 :
1132 491 : inline int OGRParquetWriterLayer::TestCapability(const char *pszCap)
1133 : {
1134 : #if PARQUET_VERSION_MAJOR <= 10
1135 : if (EQUAL(pszCap, OLCFastWriteArrowBatch))
1136 : return false;
1137 : #endif
1138 :
1139 491 : if (m_poTmpGPKGLayer && EQUAL(pszCap, OLCFastWriteArrowBatch))
1140 : {
1141 : // When using SORT_BY_BBOX=YES option, we can't directly write the
1142 : // input array, because we need to sort features. So this is not
1143 : // fast
1144 1 : return false;
1145 : }
1146 :
1147 490 : return OGRArrowWriterLayer::TestCapability(pszCap);
1148 : }
1149 :
1150 : /************************************************************************/
1151 : /* CreateFieldFromArrowSchema() */
1152 : /************************************************************************/
1153 :
1154 : #if PARQUET_VERSION_MAJOR > 10
1155 396 : bool OGRParquetWriterLayer::CreateFieldFromArrowSchema(
1156 : const struct ArrowSchema *schema, CSLConstList papszOptions)
1157 : {
1158 396 : if (m_poTmpGPKGLayer)
1159 : {
1160 : // When using SORT_BY_BBOX=YES option, we can't directly write the
1161 : // input array, because we need to sort features. But this process
1162 : // only supports the base Arrow types supported by
1163 : // OGRLayer::WriteArrowBatch()
1164 0 : return OGRLayer::CreateFieldFromArrowSchema(schema, papszOptions);
1165 : }
1166 :
1167 396 : return OGRArrowWriterLayer::CreateFieldFromArrowSchema(schema,
1168 396 : papszOptions);
1169 : }
1170 : #endif
1171 :
1172 : /************************************************************************/
1173 : /* IsArrowSchemaSupported() */
1174 : /************************************************************************/
1175 :
1176 : #if PARQUET_VERSION_MAJOR > 10
1177 1077 : bool OGRParquetWriterLayer::IsArrowSchemaSupported(
1178 : const struct ArrowSchema *schema, CSLConstList papszOptions,
1179 : std::string &osErrorMsg) const
1180 : {
1181 1077 : if (m_poTmpGPKGLayer)
1182 : {
1183 : // When using SORT_BY_BBOX=YES option, we can't directly write the
1184 : // input array, because we need to sort features. But this process
1185 : // only supports the base Arrow types supported by
1186 : // OGRLayer::WriteArrowBatch()
1187 0 : return OGRLayer::IsArrowSchemaSupported(schema, papszOptions,
1188 0 : osErrorMsg);
1189 : }
1190 :
1191 1077 : if (schema->format[0] == 'e' && schema->format[1] == 0)
1192 : {
1193 1 : osErrorMsg = "float16 not supported";
1194 1 : return false;
1195 : }
1196 1076 : if (schema->format[0] == 'v' && schema->format[1] == 'u')
1197 : {
1198 1 : osErrorMsg = "StringView not supported";
1199 1 : return false;
1200 : }
1201 1075 : if (schema->format[0] == 'v' && schema->format[1] == 'z')
1202 : {
1203 1 : osErrorMsg = "BinaryView not supported";
1204 1 : return false;
1205 : }
1206 1074 : if (schema->format[0] == '+' && schema->format[1] == 'v')
1207 : {
1208 0 : if (schema->format[2] == 'l')
1209 : {
1210 0 : osErrorMsg = "ListView not supported";
1211 0 : return false;
1212 : }
1213 0 : else if (schema->format[2] == 'L')
1214 : {
1215 0 : osErrorMsg = "LargeListView not supported";
1216 0 : return false;
1217 : }
1218 : }
1219 2136 : for (int64_t i = 0; i < schema->n_children; ++i)
1220 : {
1221 1065 : if (!IsArrowSchemaSupported(schema->children[i], papszOptions,
1222 : osErrorMsg))
1223 : {
1224 3 : return false;
1225 : }
1226 : }
1227 1071 : return true;
1228 : }
1229 : #endif
1230 :
1231 : /************************************************************************/
1232 : /* SetMetadata() */
1233 : /************************************************************************/
1234 :
1235 11 : CPLErr OGRParquetWriterLayer::SetMetadata(char **papszMetadata,
1236 : const char *pszDomain)
1237 : {
1238 11 : if (!pszDomain || !EQUAL(pszDomain, "SHAPEFILE"))
1239 : {
1240 7 : return OGRLayer::SetMetadata(papszMetadata, pszDomain);
1241 : }
1242 4 : return CE_None;
1243 : }
1244 :
1245 : /************************************************************************/
1246 : /* GetDataset() */
1247 : /************************************************************************/
1248 :
1249 23 : GDALDataset *OGRParquetWriterLayer::GetDataset()
1250 : {
1251 23 : return m_poDataset;
1252 : }
|