Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: Arrow generic code
4 : * Purpose: Arrow generic code
5 : * Author: Even Rouault, <even.rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2022, Planet Labs
9 : *
10 : * SPDX-License-Identifier: MIT
11 : ****************************************************************************/
12 :
13 : #ifndef OGARROWWRITERLAYER_HPP_INCLUDED
14 : #define OGARROWWRITERLAYER_HPP_INCLUDED
15 :
16 : #include "ogr_arrow.h"
17 :
18 : #include "cpl_json.h"
19 : #include "cpl_time.h"
20 :
21 : #include "ogrlayerarrow.h"
22 : #include "ogr_wkb.h"
23 :
24 : #include <array>
25 : #include <cinttypes>
26 : #include <limits>
27 :
28 : static constexpr int TZFLAG_UNINITIALIZED = -1;
29 :
30 : #define OGR_ARROW_RETURN_NOT_OK(status, ret_value) \
31 : do \
32 : { \
33 : if (!(status).ok()) \
34 : { \
35 : CPLError(CE_Failure, CPLE_AppDefined, "%s failed", \
36 : (status).message().c_str()); \
37 : return (ret_value); \
38 : } \
39 : } while (false)
40 :
41 : #define OGR_ARROW_RETURN_FALSE_NOT_OK(status) \
42 : OGR_ARROW_RETURN_NOT_OK(status, false)
43 :
44 : #define OGR_ARROW_RETURN_OGRERR_NOT_OK(status) \
45 : OGR_ARROW_RETURN_NOT_OK(status, OGRERR_FAILURE)
46 :
47 : #define OGR_ARROW_PROPAGATE_OGRERR(ret_value) \
48 : do \
49 : { \
50 : if ((ret_value) != OGRERR_NONE) \
51 : return OGRERR_FAILURE; \
52 : } while (0)
53 :
54 : /************************************************************************/
55 : /* OGRArrowWriterLayer() */
56 : /************************************************************************/
57 :
58 516 : inline OGRArrowWriterLayer::OGRArrowWriterLayer(
59 : arrow::MemoryPool *poMemoryPool,
60 : const std::shared_ptr<arrow::io::OutputStream> &poOutputStream,
61 516 : const char *pszLayerName)
62 516 : : m_poMemoryPool(poMemoryPool), m_poOutputStream(poOutputStream)
63 : {
64 516 : m_poFeatureDefn = new OGRFeatureDefn(pszLayerName);
65 516 : m_poFeatureDefn->SetGeomType(wkbNone);
66 516 : m_poFeatureDefn->Reference();
67 516 : SetDescription(pszLayerName);
68 516 : }
69 :
70 : /************************************************************************/
71 : /* ~OGRArrowWriterLayer() */
72 : /************************************************************************/
73 :
74 516 : inline OGRArrowWriterLayer::~OGRArrowWriterLayer()
75 : {
76 516 : CPLDebug("ARROW", "Memory pool (writer layer): bytes_allocated = %" PRId64,
77 516 : m_poMemoryPool->bytes_allocated());
78 516 : CPLDebug("ARROW", "Memory pool (writer layer): max_memory = %" PRId64,
79 516 : m_poMemoryPool->max_memory());
80 :
81 516 : m_poFeatureDefn->Release();
82 516 : }
83 :
84 : /************************************************************************/
85 : /* FinalizeWriting() */
86 : /************************************************************************/
87 :
88 504 : inline bool OGRArrowWriterLayer::FinalizeWriting()
89 : {
90 504 : bool ret = true;
91 :
92 504 : if (!IsFileWriterCreated())
93 : {
94 363 : CreateWriter();
95 : }
96 504 : if (IsFileWriterCreated())
97 : {
98 504 : PerformStepsBeforeFinalFlushGroup();
99 :
100 504 : if (!m_apoBuilders.empty() && m_apoFieldsFromArrowSchema.empty())
101 314 : ret = FlushGroup();
102 :
103 504 : if (!CloseFileWriter())
104 0 : ret = false;
105 : }
106 :
107 504 : return ret;
108 : }
109 :
110 : /************************************************************************/
111 : /* RemoveIDFromMemberOfEnsembles() */
112 : /************************************************************************/
113 :
114 : /* static */
115 : inline void
116 704 : OGRArrowWriterLayer::RemoveIDFromMemberOfEnsembles(CPLJSONObject &obj)
117 : {
118 : // Remove "id" from members of datum ensembles for compatibility with
119 : // older PROJ versions
120 : // Cf https://github.com/opengeospatial/geoparquet/discussions/110
121 : // and https://github.com/OSGeo/PROJ/pull/3221
122 704 : if (obj.GetType() == CPLJSONObject::Type::Object)
123 : {
124 898 : for (auto &subObj : obj.GetChildren())
125 : {
126 684 : RemoveIDFromMemberOfEnsembles(subObj);
127 : }
128 : }
129 542 : else if (obj.GetType() == CPLJSONObject::Type::Array &&
130 542 : obj.GetName() == "members")
131 : {
132 0 : for (auto &subObj : obj.ToArray())
133 : {
134 0 : subObj.Delete("id");
135 : }
136 : }
137 704 : }
138 :
139 : /************************************************************************/
140 : /* IdentifyCRS() */
141 : /************************************************************************/
142 :
143 : /* static */
144 : inline OGRSpatialReference
145 41 : OGRArrowWriterLayer::IdentifyCRS(const OGRSpatialReference *poSRS)
146 : {
147 41 : OGRSpatialReference oSRSIdentified(*poSRS);
148 :
149 41 : if (poSRS->GetAuthorityName(nullptr) == nullptr)
150 : {
151 : // Try to find a registered CRS that matches the input one
152 4 : int nEntries = 0;
153 4 : int *panConfidence = nullptr;
154 : OGRSpatialReferenceH *pahSRS =
155 4 : poSRS->FindMatches(nullptr, &nEntries, &panConfidence);
156 :
157 : // If there are several matches >= 90%, take the only one
158 : // that is EPSG
159 4 : int iOtherAuthority = -1;
160 4 : int iEPSG = -1;
161 4 : const char *const apszOptions[] = {
162 : "IGNORE_DATA_AXIS_TO_SRS_AXIS_MAPPING=YES", nullptr};
163 4 : int iConfidenceBestMatch = -1;
164 6 : for (int iSRS = 0; iSRS < nEntries; iSRS++)
165 : {
166 4 : auto poCandidateCRS = OGRSpatialReference::FromHandle(pahSRS[iSRS]);
167 4 : if (panConfidence[iSRS] < iConfidenceBestMatch ||
168 4 : panConfidence[iSRS] < 70)
169 : {
170 : break;
171 : }
172 3 : if (poSRS->IsSame(poCandidateCRS, apszOptions))
173 : {
174 : const char *pszAuthName =
175 3 : poCandidateCRS->GetAuthorityName(nullptr);
176 3 : if (pszAuthName != nullptr && EQUAL(pszAuthName, "EPSG"))
177 : {
178 2 : iOtherAuthority = -2;
179 2 : if (iEPSG < 0)
180 : {
181 2 : iConfidenceBestMatch = panConfidence[iSRS];
182 2 : iEPSG = iSRS;
183 : }
184 : else
185 : {
186 0 : iEPSG = -1;
187 0 : break;
188 : }
189 : }
190 1 : else if (iEPSG < 0 && pszAuthName != nullptr)
191 : {
192 1 : if (EQUAL(pszAuthName, "OGC"))
193 : {
194 : const char *pszAuthCode =
195 1 : poCandidateCRS->GetAuthorityCode(nullptr);
196 1 : if (pszAuthCode && EQUAL(pszAuthCode, "CRS84"))
197 : {
198 1 : iOtherAuthority = iSRS;
199 1 : break;
200 : }
201 : }
202 0 : else if (iOtherAuthority == -1)
203 : {
204 0 : iConfidenceBestMatch = panConfidence[iSRS];
205 0 : iOtherAuthority = iSRS;
206 : }
207 : else
208 0 : iOtherAuthority = -2;
209 : }
210 : }
211 : }
212 4 : if (iEPSG >= 0)
213 : {
214 2 : oSRSIdentified = *OGRSpatialReference::FromHandle(pahSRS[iEPSG]);
215 : }
216 2 : else if (iOtherAuthority >= 0)
217 : {
218 : oSRSIdentified =
219 1 : *OGRSpatialReference::FromHandle(pahSRS[iOtherAuthority]);
220 : }
221 4 : OSRFreeSRSArray(pahSRS);
222 4 : CPLFree(panConfidence);
223 : }
224 :
225 41 : return oSRSIdentified;
226 : }
227 :
228 : /************************************************************************/
229 : /* CreateSchemaCommon() */
230 : /************************************************************************/
231 :
232 504 : inline void OGRArrowWriterLayer::CreateSchemaCommon()
233 : {
234 504 : CPLAssert(static_cast<int>(m_aeGeomEncoding.size()) ==
235 : m_poFeatureDefn->GetGeomFieldCount());
236 :
237 1008 : std::vector<std::shared_ptr<arrow::Field>> fields;
238 504 : bool bNeedGDALSchema = false;
239 :
240 504 : m_anTZFlag.resize(m_poFeatureDefn->GetFieldCount(), TZFLAG_UNINITIALIZED);
241 :
242 504 : if (!m_osFIDColumn.empty())
243 : {
244 21 : bNeedGDALSchema = true;
245 21 : fields.emplace_back(arrow::field(m_osFIDColumn, arrow::int64(), false));
246 : }
247 :
248 504 : if (!m_apoFieldsFromArrowSchema.empty())
249 : {
250 129 : fields.insert(fields.end(), m_apoFieldsFromArrowSchema.begin(),
251 258 : m_apoFieldsFromArrowSchema.end());
252 : }
253 :
254 1272 : for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i)
255 : {
256 768 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
257 768 : std::shared_ptr<arrow::DataType> dt;
258 768 : const auto eDT = poFieldDefn->GetType();
259 768 : const auto eSubDT = poFieldDefn->GetSubType();
260 768 : const auto &osDomainName = poFieldDefn->GetDomainName();
261 768 : const OGRFieldDomain *poFieldDomain = nullptr;
262 768 : const int nWidth = poFieldDefn->GetWidth();
263 768 : if (!osDomainName.empty())
264 : {
265 4 : const auto oIter = m_oMapFieldDomains.find(osDomainName);
266 4 : if (oIter == m_oMapFieldDomains.end())
267 : {
268 0 : CPLError(CE_Warning, CPLE_AppDefined,
269 : "Field %s references domain %s, but the later one "
270 : "has not been created",
271 : poFieldDefn->GetNameRef(), osDomainName.c_str());
272 : }
273 : else
274 : {
275 4 : poFieldDomain = oIter->second.get();
276 : }
277 : }
278 768 : const char *pszFieldMetadata = nullptr;
279 768 : switch (eDT)
280 : {
281 81 : case OFTInteger:
282 81 : if (eSubDT == OFSTBoolean)
283 4 : dt = arrow::boolean();
284 77 : else if (eSubDT == OFSTInt16)
285 4 : dt = arrow::int16();
286 : else
287 73 : dt = arrow::int32();
288 81 : if (poFieldDomain != nullptr)
289 : {
290 4 : dt = arrow::dictionary(dt, arrow::utf8());
291 : }
292 81 : break;
293 :
294 51 : case OFTInteger64:
295 51 : dt = arrow::int64();
296 51 : if (poFieldDomain != nullptr)
297 : {
298 0 : dt = arrow::dictionary(dt, arrow::utf8());
299 : }
300 51 : break;
301 :
302 68 : case OFTReal:
303 : {
304 68 : const int nPrecision = poFieldDefn->GetPrecision();
305 68 : if (nWidth != 0 && nPrecision != 0)
306 : {
307 : // Since arrow 18.0, we could use arrow::smallest_decimal()
308 : // to return the smallest representation (i.e. possibly
309 : // decimal32 and decimal64). But for now keep decimal128
310 : // as the minimum for backwards compatibility.
311 : // GetValueDecimal() and other functions in
312 : // ogrlayerarrow.cpp would have to be adapted for decimal32
313 : // and decimal64 compatibility.
314 11 : if (nWidth > 38)
315 0 : dt = arrow::decimal256(nWidth, nPrecision);
316 : else
317 11 : dt = arrow::decimal128(nWidth, nPrecision);
318 : }
319 57 : else if (eSubDT == OFSTFloat32)
320 7 : dt = arrow::float32();
321 : else
322 50 : dt = arrow::float64();
323 68 : break;
324 : }
325 :
326 343 : case OFTString:
327 : case OFTWideString:
328 343 : if ((eSubDT != OFSTNone && eSubDT != OFSTJSON) || nWidth > 0)
329 3 : bNeedGDALSchema = true;
330 343 : if (eSubDT == OFSTJSON)
331 83 : pszFieldMetadata = EXTENSION_NAME_ARROW_JSON;
332 343 : dt = arrow::utf8();
333 343 : break;
334 :
335 19 : case OFTBinary:
336 19 : if (nWidth != 0)
337 4 : dt = arrow::fixed_size_binary(nWidth);
338 : else
339 15 : dt = arrow::binary();
340 19 : break;
341 :
342 48 : case OFTIntegerList:
343 48 : if (eSubDT == OFSTBoolean)
344 8 : dt = arrow::list(arrow::boolean());
345 40 : else if (eSubDT == OFSTInt16)
346 0 : dt = arrow::list(arrow::int16());
347 : else
348 40 : dt = arrow::list(arrow::int32());
349 48 : break;
350 :
351 20 : case OFTInteger64List:
352 20 : dt = arrow::list(arrow::int64());
353 20 : break;
354 :
355 35 : case OFTRealList:
356 35 : if (eSubDT == OFSTFloat32)
357 11 : dt = arrow::list(arrow::float32());
358 : else
359 24 : dt = arrow::list(arrow::float64());
360 35 : break;
361 :
362 12 : case OFTStringList:
363 : case OFTWideStringList:
364 12 : dt = arrow::list(arrow::utf8());
365 12 : break;
366 :
367 31 : case OFTDate:
368 31 : dt = arrow::date32();
369 31 : break;
370 :
371 8 : case OFTTime:
372 8 : dt = arrow::time32(arrow::TimeUnit::MILLI);
373 8 : break;
374 :
375 52 : case OFTDateTime:
376 : {
377 52 : const int nTZFlag = poFieldDefn->GetTZFlag();
378 : const char *pszTIMESTAMP_WITH_OFFSET =
379 52 : m_aosCreationOptions.FetchNameValueDef(
380 : "TIMESTAMP_WITH_OFFSET", "AUTO");
381 52 : if ((nTZFlag == OGR_TZFLAG_MIXED_TZ &&
382 5 : !EQUAL(pszTIMESTAMP_WITH_OFFSET, "NO")) ||
383 48 : EQUAL(pszTIMESTAMP_WITH_OFFSET, "YES"))
384 : {
385 4 : m_anTZFlag[i] = nTZFlag;
386 : std::vector<std::shared_ptr<arrow::Field>>
387 : tsWithOffsetFields{
388 : arrow::field(
389 : ATSWO_TIMESTAMP_FIELD_NAME,
390 8 : arrow::timestamp(arrow::TimeUnit::MILLI, "UTC"),
391 : false),
392 : arrow::field(ATSWO_OFFSET_MINUTES_FIELD_NAME,
393 24 : arrow::int16(), false)};
394 4 : dt = arrow::struct_(std::move(tsWithOffsetFields));
395 4 : pszFieldMetadata =
396 4 : EXTENSION_NAME_ARROW_TIMESTAMP_WITH_OFFSET;
397 : }
398 : else
399 : {
400 48 : if (nTZFlag >= OGR_TZFLAG_MIXED_TZ)
401 : {
402 13 : m_anTZFlag[i] = nTZFlag;
403 : }
404 48 : dt = arrow::timestamp(arrow::TimeUnit::MILLI);
405 : }
406 52 : break;
407 : }
408 : }
409 :
410 768 : auto field = arrow::field(poFieldDefn->GetNameRef(), std::move(dt),
411 2304 : CPL_TO_BOOL(poFieldDefn->IsNullable()));
412 768 : if (pszFieldMetadata)
413 : {
414 87 : auto kvMetadata = std::make_shared<arrow::KeyValueMetadata>();
415 87 : kvMetadata->Append(ARROW_EXTENSION_NAME_KEY, pszFieldMetadata);
416 87 : field = field->WithMetadata(kvMetadata);
417 : }
418 :
419 768 : fields.emplace_back(std::move(field));
420 768 : if (poFieldDefn->GetAlternativeNameRef()[0])
421 2 : bNeedGDALSchema = true;
422 768 : if (!poFieldDefn->GetComment().empty())
423 3 : bNeedGDALSchema = true;
424 : }
425 :
426 999 : for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
427 : {
428 495 : const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
429 495 : const auto eGType = poGeomFieldDefn->GetType();
430 : const int nDim =
431 495 : 2 + (OGR_GT_HasZ(eGType) ? 1 : 0) + (OGR_GT_HasM(eGType) ? 1 : 0);
432 :
433 495 : const bool pointFieldNullable = GetDriverUCName() == "PARQUET";
434 :
435 : // Fixed Size List GeoArrow encoding
436 : const auto getFixedSizeListOfPoint =
437 228 : [nDim, eGType, pointFieldNullable]()
438 : {
439 : return arrow::fixed_size_list(
440 146 : arrow::field(nDim == 2 ? "xy"
441 30 : : nDim == 3 ? (OGR_GT_HasZ(eGType) ? "xyz" : "xym")
442 : : "xyzm",
443 : arrow::float64(), pointFieldNullable),
444 116 : nDim);
445 495 : };
446 :
447 : // Struct GeoArrow encoding
448 1485 : auto xField(arrow::field("x", arrow::float64(), false));
449 1485 : auto yField(arrow::field("y", arrow::float64(), false));
450 : std::vector<std::shared_ptr<arrow::Field>> pointFields{
451 : arrow::field("x", arrow::float64(), false),
452 2970 : arrow::field("y", arrow::float64(), false)};
453 495 : if (OGR_GT_HasZ(eGType))
454 : pointFields.emplace_back(
455 145 : arrow::field("z", arrow::float64(), false));
456 495 : if (OGR_GT_HasM(eGType))
457 : pointFields.emplace_back(
458 52 : arrow::field("m", arrow::float64(), false));
459 990 : auto pointStructType(arrow::struct_(std::move(pointFields)));
460 :
461 40 : const auto getListOfVertices = [&getFixedSizeListOfPoint]()
462 : {
463 80 : return arrow::list(std::make_shared<arrow::Field>(
464 120 : "vertices", getFixedSizeListOfPoint()));
465 495 : };
466 :
467 22 : const auto getListOfRings = [&getListOfVertices]()
468 : {
469 : return arrow::list(
470 44 : std::make_shared<arrow::Field>("rings", getListOfVertices()));
471 495 : };
472 :
473 146 : const auto getListOfVerticesStruct = [&pointStructType]()
474 : {
475 : return arrow::list(
476 292 : std::make_shared<arrow::Field>("vertices", pointStructType));
477 495 : };
478 :
479 86 : const auto getListOfRingsStruct = [&getListOfVerticesStruct]()
480 : {
481 172 : return arrow::list(std::make_shared<arrow::Field>(
482 258 : "rings", getListOfVerticesStruct()));
483 495 : };
484 :
485 495 : std::shared_ptr<arrow::DataType> dt;
486 495 : switch (m_aeGeomEncoding[i])
487 : {
488 169 : case OGRArrowGeomEncoding::WKB:
489 : #if ARROW_VERSION_MAJOR >= 21
490 : if (m_bUseArrowWKBExtension)
491 : {
492 : CPLJSONDocument oMetadataDoc;
493 :
494 : const auto poSRS = poGeomFieldDefn->GetSpatialRef();
495 : if (poSRS)
496 : {
497 : OGRSpatialReference oSRSIdentified(IdentifyCRS(poSRS));
498 :
499 : // CRS encoded as PROJJSON
500 : char *pszPROJJSON = nullptr;
501 : oSRSIdentified.exportToPROJJSON(&pszPROJJSON, nullptr);
502 : CPLJSONDocument oCRSDoc;
503 : CPL_IGNORE_RET_VAL(oCRSDoc.LoadMemory(pszPROJJSON));
504 : CPLFree(pszPROJJSON);
505 : CPLJSONObject oCRSRoot = oCRSDoc.GetRoot();
506 : RemoveIDFromMemberOfEnsembles(oCRSRoot);
507 :
508 : oMetadataDoc.GetRoot().Add("crs", oCRSRoot);
509 : }
510 : else
511 : {
512 : oMetadataDoc.GetRoot().Add("crs", "srid:0");
513 : }
514 :
515 : if (m_bEdgesSpherical)
516 : {
517 : oMetadataDoc.GetRoot().Add("edges", "spherical");
518 : }
519 :
520 : const std::string metadata = oMetadataDoc.GetRoot().Format(
521 : CPLJSONObject::PrettyFormat::Plain);
522 : dt = std::make_shared<OGRGeoArrowWkbExtensionType>(
523 : arrow::binary(), metadata);
524 : }
525 : else
526 : #endif
527 : {
528 169 : dt = arrow::binary();
529 : }
530 169 : break;
531 :
532 53 : case OGRArrowGeomEncoding::WKT:
533 53 : dt = arrow::utf8();
534 53 : break;
535 :
536 0 : case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
537 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
538 0 : CPLAssert(false);
539 : break;
540 :
541 9 : case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
542 9 : dt = getFixedSizeListOfPoint();
543 9 : break;
544 :
545 9 : case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
546 9 : dt = getListOfVertices();
547 9 : break;
548 :
549 11 : case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
550 11 : dt = getListOfRings();
551 11 : break;
552 :
553 9 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
554 18 : dt = arrow::list(std::make_shared<arrow::Field>(
555 27 : "points", getFixedSizeListOfPoint()));
556 9 : break;
557 :
558 9 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
559 18 : dt = arrow::list(std::make_shared<arrow::Field>(
560 27 : "linestrings", getListOfVertices()));
561 9 : break;
562 :
563 11 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
564 22 : dt = arrow::list(std::make_shared<arrow::Field>(
565 33 : "polygons", getListOfRings()));
566 11 : break;
567 :
568 39 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
569 39 : dt = pointStructType;
570 39 : break;
571 :
572 30 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
573 30 : dt = getListOfVerticesStruct();
574 30 : break;
575 :
576 44 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
577 44 : dt = getListOfRingsStruct();
578 44 : break;
579 :
580 30 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
581 60 : dt = arrow::list(
582 90 : std::make_shared<arrow::Field>("points", pointStructType));
583 30 : break;
584 :
585 30 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
586 60 : dt = arrow::list(std::make_shared<arrow::Field>(
587 90 : "linestrings", getListOfVerticesStruct()));
588 30 : break;
589 :
590 42 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
591 84 : dt = arrow::list(std::make_shared<arrow::Field>(
592 126 : "polygons", getListOfRingsStruct()));
593 42 : break;
594 : }
595 :
596 : std::shared_ptr<arrow::Field> field(
597 495 : arrow::field(poGeomFieldDefn->GetNameRef(), std::move(dt),
598 1485 : CPL_TO_BOOL(poGeomFieldDefn->IsNullable())));
599 495 : if (m_bWriteFieldArrowExtensionName)
600 : {
601 138 : auto kvMetadata = field->metadata()
602 138 : ? field->metadata()->Copy()
603 138 : : std::make_shared<arrow::KeyValueMetadata>();
604 276 : kvMetadata->Append(
605 : ARROW_EXTENSION_NAME_KEY,
606 138 : GetGeomEncodingAsString(m_aeGeomEncoding[i], false));
607 138 : field = field->WithMetadata(kvMetadata);
608 : }
609 :
610 495 : m_apoBaseStructGeomType.emplace_back(std::move(pointStructType));
611 :
612 495 : fields.emplace_back(std::move(field));
613 : }
614 :
615 504 : if (m_bWriteBBoxStruct)
616 : {
617 579 : for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
618 : {
619 288 : const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
620 864 : auto bbox_field_xmin(arrow::field("xmin", arrow::float32(), false));
621 864 : auto bbox_field_ymin(arrow::field("ymin", arrow::float32(), false));
622 864 : auto bbox_field_xmax(arrow::field("xmax", arrow::float32(), false));
623 864 : auto bbox_field_ymax(arrow::field("ymax", arrow::float32(), false));
624 : auto bbox_field(arrow::field(
625 288 : m_oBBoxStructFieldName.empty()
626 963 : ? std::string(poGeomFieldDefn->GetNameRef()).append("_bbox")
627 : : m_oBBoxStructFieldName,
628 1728 : arrow::struct_(
629 288 : {std::move(bbox_field_xmin), std::move(bbox_field_ymin),
630 1728 : std::move(bbox_field_xmax), std::move(bbox_field_ymax)}),
631 1602 : CPL_TO_BOOL(poGeomFieldDefn->IsNullable())));
632 288 : fields.emplace_back(bbox_field);
633 288 : m_apoFieldsBBOX.emplace_back(bbox_field);
634 : }
635 : }
636 :
637 504 : m_aoEnvelopes.resize(m_poFeatureDefn->GetGeomFieldCount());
638 504 : m_oSetWrittenGeometryTypes.resize(m_poFeatureDefn->GetGeomFieldCount());
639 :
640 504 : m_poSchema = arrow::schema(std::move(fields));
641 504 : CPLAssert(m_poSchema);
642 531 : if (bNeedGDALSchema &&
643 27 : CPLTestBool(CPLGetConfigOption(
644 531 : ("OGR_" + GetDriverUCName() + "_WRITE_GDAL_SCHEMA").c_str(),
645 : "YES")))
646 : {
647 54 : CPLJSONObject oRoot;
648 54 : CPLJSONObject oColumns;
649 :
650 27 : if (!m_osFIDColumn.empty())
651 21 : oRoot.Add("fid", m_osFIDColumn);
652 :
653 27 : oRoot.Add("columns", oColumns);
654 221 : for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i)
655 : {
656 194 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
657 388 : CPLJSONObject oColumn;
658 194 : oColumns.Add(poFieldDefn->GetNameRef(), oColumn);
659 194 : oColumn.Add("type", OGR_GetFieldTypeName(poFieldDefn->GetType()));
660 194 : const auto eSubDT = poFieldDefn->GetSubType();
661 194 : if (eSubDT != OFSTNone)
662 56 : oColumn.Add("subtype", OGR_GetFieldSubTypeName(eSubDT));
663 194 : const int nWidth = poFieldDefn->GetWidth();
664 194 : if (nWidth > 0)
665 15 : oColumn.Add("width", nWidth);
666 194 : const int nPrecision = poFieldDefn->GetPrecision();
667 194 : if (nPrecision > 0)
668 7 : oColumn.Add("precision", nPrecision);
669 194 : if (poFieldDefn->GetAlternativeNameRef()[0])
670 2 : oColumn.Add("alternative_name",
671 : poFieldDefn->GetAlternativeNameRef());
672 194 : if (!poFieldDefn->GetComment().empty())
673 3 : oColumn.Add("comment", poFieldDefn->GetComment());
674 : }
675 :
676 27 : auto kvMetadata = m_poSchema->metadata()
677 0 : ? m_poSchema->metadata()->Copy()
678 54 : : std::make_shared<arrow::KeyValueMetadata>();
679 54 : kvMetadata->Append("gdal:schema",
680 54 : oRoot.Format(CPLJSONObject::PrettyFormat::Plain));
681 27 : m_poSchema = m_poSchema->WithMetadata(kvMetadata);
682 27 : CPLAssert(m_poSchema);
683 : }
684 504 : }
685 :
686 : /************************************************************************/
687 : /* FinalizeSchema() */
688 : /************************************************************************/
689 :
690 446 : inline void OGRArrowWriterLayer::FinalizeSchema()
691 : {
692 : // Final tuning of schema taking into actual timezone values
693 : // from features
694 446 : int nArrowIdxFirstField = !m_osFIDColumn.empty() ? 1 : 0;
695 1210 : for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i)
696 : {
697 764 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
698 778 : if (m_anTZFlag[i] >= OGR_TZFLAG_MIXED_TZ &&
699 778 : poFieldDefn->GetTZFlag() != OGR_TZFLAG_MIXED_TZ &&
700 12 : m_poSchema->field(nArrowIdxFirstField + i)->type()->id() !=
701 : arrow::Type::STRUCT)
702 : {
703 12 : const int nOffset = m_anTZFlag[i] == OGR_TZFLAG_MIXED_TZ
704 12 : ? 0
705 12 : : (m_anTZFlag[i] - OGR_TZFLAG_UTC) * 15;
706 12 : int nHours = static_cast<int>(nOffset / 60); // Round towards zero.
707 12 : const int nMinutes = std::abs(nOffset - nHours * 60);
708 :
709 : const std::string osTZ =
710 : CPLSPrintf("%c%02d:%02d", nOffset >= 0 ? '+' : '-',
711 24 : std::abs(nHours), nMinutes);
712 24 : auto dt = arrow::timestamp(arrow::TimeUnit::MILLI, osTZ);
713 12 : auto field = arrow::field(poFieldDefn->GetNameRef(), std::move(dt),
714 36 : CPL_TO_BOOL(poFieldDefn->IsNullable()));
715 24 : auto result = m_poSchema->SetField(nArrowIdxFirstField + i, field);
716 12 : if (!result.ok())
717 : {
718 0 : CPLError(CE_Warning, CPLE_AppDefined,
719 : "Schema::SetField() failed with %s",
720 0 : result.status().message().c_str());
721 : }
722 : else
723 : {
724 12 : m_poSchema = *result;
725 : }
726 : }
727 : }
728 446 : }
729 :
730 : /************************************************************************/
731 : /* AddFieldDomain() */
732 : /************************************************************************/
733 :
734 : inline bool
735 11 : OGRArrowWriterLayer::AddFieldDomain(std::unique_ptr<OGRFieldDomain> &&domain,
736 : std::string &failureReason)
737 : {
738 11 : if (domain->GetDomainType() != OFDT_CODED)
739 : {
740 0 : failureReason = "Only coded field domains are supported by Arrow";
741 0 : return false;
742 : }
743 :
744 : const OGRCodedFieldDomain *poDomain =
745 11 : static_cast<const OGRCodedFieldDomain *>(domain.get());
746 11 : const OGRCodedValue *psIter = poDomain->GetEnumeration();
747 :
748 : auto poStringBuilder =
749 22 : std::make_shared<arrow::StringBuilder>(m_poMemoryPool);
750 :
751 11 : int nLastCode = -1;
752 44 : for (; psIter->pszCode; ++psIter)
753 : {
754 33 : if (CPLGetValueType(psIter->pszCode) != CPL_VALUE_INTEGER)
755 : {
756 0 : failureReason = "Non integer code in domain ";
757 0 : failureReason += domain->GetName();
758 0 : return false;
759 : }
760 33 : int nCode = atoi(psIter->pszCode);
761 33 : if (nCode <= nLastCode || nCode - nLastCode > 100)
762 : {
763 0 : failureReason = "Too sparse codes in domain ";
764 0 : failureReason += domain->GetName();
765 0 : return false;
766 : }
767 33 : for (int i = nLastCode + 1; i < nCode; ++i)
768 : {
769 0 : OGR_ARROW_RETURN_FALSE_NOT_OK(poStringBuilder->AppendNull());
770 : }
771 33 : if (psIter->pszValue)
772 33 : OGR_ARROW_RETURN_FALSE_NOT_OK(
773 : poStringBuilder->Append(psIter->pszValue));
774 : else
775 0 : OGR_ARROW_RETURN_FALSE_NOT_OK(poStringBuilder->AppendNull());
776 33 : nLastCode = nCode;
777 : }
778 :
779 11 : std::shared_ptr<arrow::Array> stringArray;
780 22 : auto status = poStringBuilder->Finish(&stringArray);
781 11 : if (!status.ok())
782 : {
783 0 : CPLError(CE_Failure, CPLE_AppDefined,
784 : "StringArray::Finish() failed with %s",
785 0 : status.message().c_str());
786 0 : return false;
787 : }
788 :
789 11 : m_oMapFieldDomainToStringArray[domain->GetName()] = std::move(stringArray);
790 11 : m_oMapFieldDomains[domain->GetName()] = std::move(domain);
791 11 : return true;
792 : }
793 :
794 : /************************************************************************/
795 : /* GetFieldDomainNames() */
796 : /************************************************************************/
797 :
798 0 : inline std::vector<std::string> OGRArrowWriterLayer::GetFieldDomainNames() const
799 : {
800 0 : std::vector<std::string> names;
801 0 : names.reserve(m_oMapFieldDomains.size());
802 0 : for (const auto &it : m_oMapFieldDomains)
803 : {
804 0 : names.emplace_back(it.first);
805 : }
806 0 : return names;
807 : }
808 :
809 : /************************************************************************/
810 : /* GetFieldDomain() */
811 : /************************************************************************/
812 :
813 : inline const OGRFieldDomain *
814 15 : OGRArrowWriterLayer::GetFieldDomain(const std::string &name) const
815 : {
816 15 : const auto iter = m_oMapFieldDomains.find(name);
817 15 : if (iter == m_oMapFieldDomains.end())
818 11 : return nullptr;
819 4 : return iter->second.get();
820 : }
821 :
822 : /************************************************************************/
823 : /* CreateField() */
824 : /************************************************************************/
825 :
826 769 : inline OGRErr OGRArrowWriterLayer::CreateField(const OGRFieldDefn *poField,
827 : int /* bApproxOK */)
828 : {
829 769 : if (m_poSchema)
830 : {
831 1 : CPLError(CE_Failure, CPLE_NotSupported,
832 : "Cannot add field after a first feature has been written");
833 1 : return OGRERR_FAILURE;
834 : }
835 768 : if (!m_apoFieldsFromArrowSchema.empty())
836 : {
837 0 : CPLError(CE_Failure, CPLE_NotSupported,
838 : "Cannot mix calls to CreateField() and "
839 : "CreateFieldFromArrowSchema()");
840 0 : return OGRERR_FAILURE;
841 : }
842 768 : m_poFeatureDefn->AddFieldDefn(poField);
843 768 : return OGRERR_NONE;
844 : }
845 :
846 : /************************************************************************/
847 : /* OGRLayer::CreateFieldFromArrowSchema() */
848 : /************************************************************************/
849 :
850 1081 : inline bool OGRArrowWriterLayer::CreateFieldFromArrowSchema(
851 : const struct ArrowSchema *schema, CSLConstList /*papszOptions*/)
852 : {
853 1081 : if (m_poSchema)
854 : {
855 0 : CPLError(CE_Failure, CPLE_NotSupported,
856 : "Cannot add field after a first feature has been written");
857 0 : return false;
858 : }
859 :
860 1081 : if (m_poFeatureDefn->GetFieldCount())
861 : {
862 0 : CPLError(CE_Failure, CPLE_NotSupported,
863 : "Cannot mix calls to CreateField() and "
864 : "CreateFieldFromArrowSchema()");
865 0 : return false;
866 : }
867 :
868 1081 : if (m_osFIDColumn == schema->name)
869 : {
870 0 : CPLError(CE_Failure, CPLE_AppDefined,
871 : "FID column has the same name as this field: %s",
872 0 : schema->name);
873 0 : return false;
874 : }
875 :
876 36422 : for (auto &apoField : m_apoFieldsFromArrowSchema)
877 : {
878 35341 : if (apoField->name() == schema->name)
879 : {
880 0 : CPLError(CE_Failure, CPLE_AppDefined,
881 0 : "Field of name %s already exists", schema->name);
882 0 : return false;
883 : }
884 : }
885 :
886 1081 : if (m_poFeatureDefn->GetGeomFieldIndex(schema->name) >= 0)
887 : {
888 0 : CPLError(CE_Failure, CPLE_AppDefined,
889 0 : "Geometry field of name %s already exists", schema->name);
890 0 : return false;
891 : }
892 :
893 : // ImportField() would release the schema, but we don't want that
894 : // So copy the structure content into a local variable, and override its
895 : // release callback to a no-op. This may be a bit fragile, but it doesn't
896 : // look like ImportField implementation tries to access the C ArrowSchema
897 : // after it has been called.
898 1081 : struct ArrowSchema lSchema = *schema;
899 1081 : const auto DummyFreeSchema = [](struct ArrowSchema *ptrSchema)
900 1081 : { ptrSchema->release = nullptr; };
901 1081 : lSchema.release = DummyFreeSchema;
902 2162 : auto result = arrow::ImportField(&lSchema);
903 1081 : CPLAssert(lSchema.release == nullptr);
904 1081 : if (!result.ok())
905 : {
906 0 : CPLError(CE_Failure, CPLE_AppDefined,
907 : "CreateFieldFromArrowSchema() failed: %s",
908 0 : result.status().message().c_str());
909 0 : return false;
910 : }
911 1081 : m_apoFieldsFromArrowSchema.emplace_back(std::move(*result));
912 1081 : return true;
913 : }
914 :
915 : /************************************************************************/
916 : /* GetPreciseArrowGeomEncoding() */
917 : /************************************************************************/
918 :
919 275 : inline OGRArrowGeomEncoding OGRArrowWriterLayer::GetPreciseArrowGeomEncoding(
920 : OGRArrowGeomEncoding eEncodingType, OGRwkbGeometryType eGType)
921 : {
922 275 : CPLAssert(eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC ||
923 : eEncodingType == OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC);
924 275 : const auto eFlatType = wkbFlatten(eGType);
925 275 : if (eFlatType == wkbPoint)
926 : {
927 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
928 48 : ? OGRArrowGeomEncoding::GEOARROW_FSL_POINT
929 48 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT;
930 : }
931 227 : else if (eFlatType == wkbLineString)
932 : {
933 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
934 39 : ? OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING
935 39 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING;
936 : }
937 188 : else if (eFlatType == wkbPolygon)
938 : {
939 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
940 55 : ? OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON
941 55 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON;
942 : }
943 133 : else if (eFlatType == wkbMultiPoint)
944 : {
945 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
946 39 : ? OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT
947 39 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT;
948 : }
949 94 : else if (eFlatType == wkbMultiLineString)
950 : {
951 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
952 39 : ? OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING
953 39 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING;
954 : }
955 55 : else if (eFlatType == wkbMultiPolygon)
956 : {
957 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
958 53 : ? OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON
959 53 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON;
960 : }
961 : else
962 : {
963 2 : CPLError(CE_Failure, CPLE_NotSupported,
964 : "GeoArrow encoding is currently not supported for %s",
965 : OGRGeometryTypeToName(eGType));
966 2 : return eEncodingType;
967 : }
968 : }
969 :
970 : /************************************************************************/
971 : /* GetGeomEncodingAsString() */
972 : /************************************************************************/
973 :
974 : inline const char *
975 747 : OGRArrowWriterLayer::GetGeomEncodingAsString(OGRArrowGeomEncoding eGeomEncoding,
976 : bool bForParquetGeo)
977 : {
978 747 : switch (eGeomEncoding)
979 : {
980 216 : case OGRArrowGeomEncoding::WKB:
981 216 : return bForParquetGeo ? "WKB" : "geoarrow.wkb";
982 111 : case OGRArrowGeomEncoding::WKT:
983 111 : return bForParquetGeo ? "WKT" : "geoarrow.wkt";
984 0 : case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
985 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
986 0 : CPLAssert(false);
987 : break;
988 19 : case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
989 19 : return "geoarrow.point";
990 19 : case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
991 19 : return "geoarrow.linestring";
992 21 : case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
993 21 : return "geoarrow.polygon";
994 19 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
995 19 : return "geoarrow.multipoint";
996 19 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
997 19 : return "geoarrow.multilinestring";
998 21 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
999 21 : return "geoarrow.multipolygon";
1000 62 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
1001 62 : return bForParquetGeo ? "point" : "geoarrow.point";
1002 42 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
1003 42 : return bForParquetGeo ? "linestring" : "geoarrow.linestring";
1004 60 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
1005 60 : return bForParquetGeo ? "polygon" : "geoarrow.polygon";
1006 42 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
1007 42 : return bForParquetGeo ? "multipoint" : "geoarrow.multipoint";
1008 42 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
1009 42 : return bForParquetGeo ? "multilinestring"
1010 42 : : "geoarrow.multilinestring";
1011 54 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
1012 54 : return bForParquetGeo ? "multipolygon" : "geoarrow.multipolygon";
1013 : }
1014 0 : return nullptr;
1015 : }
1016 :
1017 : /************************************************************************/
1018 : /* CreateGeomField() */
1019 : /************************************************************************/
1020 :
1021 : inline OGRErr
1022 27 : OGRArrowWriterLayer::CreateGeomField(const OGRGeomFieldDefn *poField,
1023 : int /* bApproxOK */)
1024 : {
1025 27 : if (m_poSchema)
1026 : {
1027 1 : CPLError(CE_Failure, CPLE_NotSupported,
1028 : "Cannot add field after a first feature has been written");
1029 1 : return OGRERR_FAILURE;
1030 : }
1031 26 : const auto eGType = poField->GetType();
1032 26 : if (!IsSupportedGeometryType(eGType))
1033 : {
1034 0 : return OGRERR_FAILURE;
1035 : }
1036 :
1037 26 : if (IsSRSRequired() && poField->GetSpatialRef() == nullptr)
1038 : {
1039 0 : CPLError(CE_Warning, CPLE_AppDefined,
1040 : "Geometry column should have an associated CRS");
1041 : }
1042 26 : auto eGeomEncoding = m_eGeomEncoding;
1043 26 : if (eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC ||
1044 26 : eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC)
1045 : {
1046 0 : const auto eEncodingType = eGeomEncoding;
1047 0 : eGeomEncoding = GetPreciseArrowGeomEncoding(eEncodingType, eGType);
1048 0 : if (eGeomEncoding == eEncodingType)
1049 0 : return OGRERR_FAILURE;
1050 : }
1051 26 : m_aeGeomEncoding.push_back(eGeomEncoding);
1052 26 : m_poFeatureDefn->AddGeomFieldDefn(poField);
1053 26 : return OGRERR_NONE;
1054 : }
1055 :
1056 : /************************************************************************/
1057 : /* MakeGeoArrowBuilder() */
1058 : /************************************************************************/
1059 :
1060 : static std::shared_ptr<arrow::ArrayBuilder>
1061 134 : MakeGeoArrowBuilder(arrow::MemoryPool *poMemoryPool, int nDim, int nDepth)
1062 : {
1063 134 : if (nDepth == 0)
1064 104 : return std::make_shared<arrow::FixedSizeListBuilder>(
1065 104 : poMemoryPool, std::make_shared<arrow::DoubleBuilder>(poMemoryPool),
1066 52 : nDim);
1067 : else
1068 164 : return std::make_shared<arrow::ListBuilder>(
1069 246 : poMemoryPool, MakeGeoArrowBuilder(poMemoryPool, nDim, nDepth - 1));
1070 : }
1071 :
1072 : /************************************************************************/
1073 : /* MakeGeoArrowStructBuilder() */
1074 : /************************************************************************/
1075 :
1076 : static std::shared_ptr<arrow::ArrayBuilder>
1077 538 : MakeGeoArrowStructBuilder(arrow::MemoryPool *poMemoryPool, int nDim, int nDepth,
1078 : const std::shared_ptr<arrow::DataType> &eBaseType)
1079 : {
1080 538 : if (nDepth == 0)
1081 : {
1082 213 : std::vector<std::shared_ptr<arrow::ArrayBuilder>> builders;
1083 735 : for (int i = 0; i < nDim; ++i)
1084 : builders.emplace_back(
1085 522 : std::make_shared<arrow::DoubleBuilder>(poMemoryPool));
1086 426 : return std::make_shared<arrow::StructBuilder>(eBaseType, poMemoryPool,
1087 426 : std::move(builders));
1088 : }
1089 : else
1090 650 : return std::make_shared<arrow::ListBuilder>(
1091 650 : poMemoryPool, MakeGeoArrowStructBuilder(poMemoryPool, nDim,
1092 325 : nDepth - 1, eBaseType));
1093 : }
1094 :
1095 : /************************************************************************/
1096 : /* ClearArrayBuilers() */
1097 : /************************************************************************/
1098 :
1099 355 : inline void OGRArrowWriterLayer::ClearArrayBuilers()
1100 : {
1101 355 : m_apoBuilders.clear();
1102 355 : m_apoBuildersBBOXStruct.clear();
1103 355 : m_apoBuildersBBOXXMin.clear();
1104 355 : m_apoBuildersBBOXYMin.clear();
1105 355 : m_apoBuildersBBOXXMax.clear();
1106 355 : m_apoBuildersBBOXYMax.clear();
1107 355 : }
1108 :
1109 : /************************************************************************/
1110 : /* CreateArrayBuilders() */
1111 : /************************************************************************/
1112 :
1113 484 : inline void OGRArrowWriterLayer::CreateArrayBuilders()
1114 : {
1115 484 : m_apoBuilders.reserve(1 + m_poFeatureDefn->GetFieldCount() +
1116 484 : m_poFeatureDefn->GetGeomFieldCount());
1117 :
1118 484 : int nArrowIdx = 0;
1119 484 : if (!m_osFIDColumn.empty())
1120 : {
1121 51 : m_apoBuilders.emplace_back(std::make_shared<arrow::Int64Builder>());
1122 51 : nArrowIdx++;
1123 : }
1124 :
1125 1966 : for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i, ++nArrowIdx)
1126 : {
1127 1482 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
1128 1482 : std::shared_ptr<arrow::ArrayBuilder> builder;
1129 1482 : const auto eSubDT = poFieldDefn->GetSubType();
1130 1482 : switch (poFieldDefn->GetType())
1131 : {
1132 162 : case OFTInteger:
1133 162 : if (eSubDT == OFSTBoolean)
1134 : builder =
1135 12 : std::make_shared<arrow::BooleanBuilder>(m_poMemoryPool);
1136 150 : else if (eSubDT == OFSTInt16)
1137 : builder =
1138 12 : std::make_shared<arrow::Int16Builder>(m_poMemoryPool);
1139 : else
1140 : builder =
1141 138 : std::make_shared<arrow::Int32Builder>(m_poMemoryPool);
1142 162 : break;
1143 :
1144 99 : case OFTInteger64:
1145 99 : builder = std::make_shared<arrow::Int64Builder>(m_poMemoryPool);
1146 99 : break;
1147 :
1148 122 : case OFTReal:
1149 : {
1150 244 : const auto arrowType = m_poSchema->fields()[nArrowIdx]->type();
1151 122 : if (arrowType->id() == arrow::Type::DECIMAL128)
1152 27 : builder = std::make_shared<arrow::Decimal128Builder>(
1153 27 : arrowType, m_poMemoryPool);
1154 95 : else if (arrowType->id() == arrow::Type::DECIMAL256)
1155 0 : builder = std::make_shared<arrow::Decimal256Builder>(
1156 0 : arrowType, m_poMemoryPool);
1157 95 : else if (eSubDT == OFSTFloat32)
1158 : builder =
1159 21 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool);
1160 : else
1161 : builder =
1162 74 : std::make_shared<arrow::DoubleBuilder>(m_poMemoryPool);
1163 122 : break;
1164 : }
1165 :
1166 540 : case OFTString:
1167 : case OFTWideString:
1168 : builder =
1169 540 : std::make_shared<arrow::StringBuilder>(m_poMemoryPool);
1170 540 : break;
1171 :
1172 43 : case OFTBinary:
1173 43 : if (poFieldDefn->GetWidth() != 0)
1174 24 : builder = std::make_shared<arrow::FixedSizeBinaryBuilder>(
1175 24 : arrow::fixed_size_binary(poFieldDefn->GetWidth()),
1176 24 : m_poMemoryPool);
1177 : else
1178 : builder =
1179 31 : std::make_shared<arrow::BinaryBuilder>(m_poMemoryPool);
1180 43 : break;
1181 :
1182 144 : case OFTIntegerList:
1183 : {
1184 144 : std::shared_ptr<arrow::ArrayBuilder> poBaseBuilder;
1185 144 : if (eSubDT == OFSTBoolean)
1186 : poBaseBuilder =
1187 24 : std::make_shared<arrow::BooleanBuilder>(m_poMemoryPool);
1188 120 : else if (eSubDT == OFSTInt16)
1189 : poBaseBuilder =
1190 0 : std::make_shared<arrow::Int16Builder>(m_poMemoryPool);
1191 : else
1192 : poBaseBuilder =
1193 120 : std::make_shared<arrow::Int32Builder>(m_poMemoryPool);
1194 288 : builder = std::make_shared<arrow::ListBuilder>(m_poMemoryPool,
1195 144 : poBaseBuilder);
1196 144 : break;
1197 : }
1198 :
1199 60 : case OFTInteger64List:
1200 60 : builder = std::make_shared<arrow::ListBuilder>(
1201 60 : m_poMemoryPool,
1202 180 : std::make_shared<arrow::Int64Builder>(m_poMemoryPool));
1203 :
1204 60 : break;
1205 :
1206 105 : case OFTRealList:
1207 105 : if (eSubDT == OFSTFloat32)
1208 33 : builder = std::make_shared<arrow::ListBuilder>(
1209 33 : m_poMemoryPool,
1210 99 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1211 : else
1212 72 : builder = std::make_shared<arrow::ListBuilder>(
1213 72 : m_poMemoryPool,
1214 216 : std::make_shared<arrow::DoubleBuilder>(m_poMemoryPool));
1215 105 : break;
1216 :
1217 36 : case OFTStringList:
1218 : case OFTWideStringList:
1219 36 : builder = std::make_shared<arrow::ListBuilder>(
1220 36 : m_poMemoryPool,
1221 108 : std::make_shared<arrow::StringBuilder>(m_poMemoryPool));
1222 :
1223 36 : break;
1224 :
1225 47 : case OFTDate:
1226 : builder =
1227 47 : std::make_shared<arrow::Date32Builder>(m_poMemoryPool);
1228 47 : break;
1229 :
1230 24 : case OFTTime:
1231 48 : builder = std::make_shared<arrow::Time32Builder>(
1232 72 : arrow::time32(arrow::TimeUnit::MILLI), m_poMemoryPool);
1233 24 : break;
1234 :
1235 100 : case OFTDateTime:
1236 : {
1237 200 : const auto arrowType = m_poSchema->fields()[nArrowIdx]->type();
1238 100 : if (arrowType->id() == arrow::Type::STRUCT)
1239 : {
1240 4 : builder = std::make_shared<arrow::StructBuilder>(
1241 4 : arrowType, m_poMemoryPool,
1242 20 : std::vector<std::shared_ptr<arrow::ArrayBuilder>>{
1243 4 : std::make_shared<arrow::TimestampBuilder>(
1244 4 : arrow::timestamp(arrow::TimeUnit::MILLI),
1245 4 : m_poMemoryPool),
1246 4 : std::make_shared<arrow::Int16Builder>(
1247 16 : m_poMemoryPool)});
1248 : }
1249 : else
1250 : {
1251 192 : builder = std::make_shared<arrow::TimestampBuilder>(
1252 96 : arrow::timestamp(arrow::TimeUnit::MILLI),
1253 192 : m_poMemoryPool);
1254 : }
1255 100 : break;
1256 : }
1257 : }
1258 1482 : m_apoBuilders.emplace_back(builder);
1259 : }
1260 :
1261 959 : for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i, ++nArrowIdx)
1262 : {
1263 475 : std::shared_ptr<arrow::ArrayBuilder> builder;
1264 475 : const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
1265 475 : const auto eGType = poGeomFieldDefn->GetType();
1266 : const int nDim =
1267 475 : 2 + (OGR_GT_HasZ(eGType) ? 1 : 0) + (OGR_GT_HasM(eGType) ? 1 : 0);
1268 :
1269 475 : switch (m_aeGeomEncoding[i])
1270 : {
1271 157 : case OGRArrowGeomEncoding::WKB:
1272 : builder =
1273 157 : std::make_shared<arrow::BinaryBuilder>(m_poMemoryPool);
1274 157 : break;
1275 :
1276 53 : case OGRArrowGeomEncoding::WKT:
1277 : builder =
1278 53 : std::make_shared<arrow::StringBuilder>(m_poMemoryPool);
1279 53 : break;
1280 :
1281 8 : case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
1282 8 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 0);
1283 8 : break;
1284 :
1285 8 : case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
1286 8 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 1);
1287 8 : break;
1288 :
1289 10 : case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
1290 10 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 2);
1291 10 : break;
1292 :
1293 8 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
1294 8 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 1);
1295 8 : break;
1296 :
1297 8 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
1298 8 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 2);
1299 8 : break;
1300 :
1301 10 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
1302 10 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 3);
1303 10 : break;
1304 :
1305 42 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
1306 84 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 0,
1307 84 : m_apoBaseStructGeomType[i]);
1308 42 : break;
1309 :
1310 29 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
1311 58 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 1,
1312 58 : m_apoBaseStructGeomType[i]);
1313 29 : break;
1314 :
1315 43 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
1316 86 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 2,
1317 86 : m_apoBaseStructGeomType[i]);
1318 43 : break;
1319 :
1320 29 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
1321 58 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 1,
1322 58 : m_apoBaseStructGeomType[i]);
1323 29 : break;
1324 :
1325 29 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
1326 58 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 2,
1327 58 : m_apoBaseStructGeomType[i]);
1328 29 : break;
1329 :
1330 41 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
1331 82 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 3,
1332 82 : m_apoBaseStructGeomType[i]);
1333 41 : break;
1334 :
1335 0 : case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
1336 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
1337 0 : CPLAssert(false);
1338 : break;
1339 : }
1340 :
1341 475 : m_apoBuilders.emplace_back(builder);
1342 :
1343 475 : if (m_bWriteBBoxStruct)
1344 : {
1345 : m_apoBuildersBBOXXMin.emplace_back(
1346 276 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1347 : m_apoBuildersBBOXYMin.emplace_back(
1348 276 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1349 : m_apoBuildersBBOXXMax.emplace_back(
1350 276 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1351 : m_apoBuildersBBOXYMax.emplace_back(
1352 276 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1353 : m_apoBuildersBBOXStruct.emplace_back(
1354 552 : std::make_shared<arrow::StructBuilder>(
1355 276 : m_apoFieldsBBOX[i]->type(), m_poMemoryPool,
1356 2484 : std::vector<std::shared_ptr<arrow::ArrayBuilder>>{
1357 276 : m_apoBuildersBBOXXMin.back(),
1358 276 : m_apoBuildersBBOXYMin.back(),
1359 276 : m_apoBuildersBBOXXMax.back(),
1360 1932 : m_apoBuildersBBOXYMax.back()}));
1361 : }
1362 : }
1363 484 : }
1364 :
1365 : /************************************************************************/
1366 : /* castToFloatDown() */
1367 : /************************************************************************/
1368 :
1369 : // Cf https://github.com/sqlite/sqlite/blob/90e4a3b7fcdf63035d6f35eb44d11ff58ff4b068/ext/rtree/rtree.c#L2993C1-L2995C3
1370 : /*
1371 : ** Rounding constants for float->double conversion.
1372 : */
1373 : #define RNDTOWARDS (1.0 - 1.0 / 8388608.0) /* Round towards zero */
1374 : #define RNDAWAY (1.0 + 1.0 / 8388608.0) /* Round away from zero */
1375 :
1376 : /*
1377 : ** Convert an sqlite3_value into an RtreeValue (presumably a float)
1378 : ** while taking care to round toward negative or positive, respectively.
1379 : */
1380 3836 : static float castToFloatDown(double d)
1381 : {
1382 3836 : float f = static_cast<float>(d);
1383 3836 : if (f > d)
1384 : {
1385 22 : f = static_cast<float>(d * (d < 0 ? RNDAWAY : RNDTOWARDS));
1386 : }
1387 3836 : return f;
1388 : }
1389 :
1390 3836 : static float castToFloatUp(double d)
1391 : {
1392 3836 : float f = static_cast<float>(d);
1393 3836 : if (f < d)
1394 : {
1395 12 : f = static_cast<float>(d * (d < 0 ? RNDTOWARDS : RNDAWAY));
1396 : }
1397 3836 : return f;
1398 : }
1399 :
1400 : /************************************************************************/
1401 : /* GeoArrowLineBuilder() */
1402 : /************************************************************************/
1403 :
1404 : template <class PointBuilderType>
1405 676 : static OGRErr GeoArrowLineBuilder(const OGRLineString *poLS,
1406 : PointBuilderType *poPointBuilder,
1407 : arrow::DoubleBuilder *poXBuilder,
1408 : arrow::DoubleBuilder *poYBuilder,
1409 : arrow::DoubleBuilder *poZBuilder,
1410 : arrow::DoubleBuilder *poMBuilder)
1411 : {
1412 3076 : for (int j = 0; j < poLS->getNumPoints(); ++j)
1413 : {
1414 2400 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1415 2400 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poXBuilder->Append(poLS->getX(j)));
1416 2400 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poYBuilder->Append(poLS->getY(j)));
1417 2400 : if (poZBuilder)
1418 668 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poZBuilder->Append(poLS->getZ(j)));
1419 2400 : if (poMBuilder)
1420 220 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMBuilder->Append(poLS->getM(j)));
1421 : }
1422 676 : return OGRERR_NONE;
1423 : }
1424 :
1425 : /************************************************************************/
1426 : /* BuildGeometry() */
1427 : /************************************************************************/
1428 :
1429 3892 : inline OGRErr OGRArrowWriterLayer::BuildGeometry(OGRGeometry *poGeom,
1430 : int iGeomField,
1431 : arrow::ArrayBuilder *poBuilder)
1432 : {
1433 3892 : const auto eGType = poGeom ? poGeom->getGeometryType() : wkbNone;
1434 : const auto eColumnGType =
1435 3892 : m_poFeatureDefn->GetGeomFieldDefn(iGeomField)->GetType();
1436 3892 : const bool bHasZ = CPL_TO_BOOL(OGR_GT_HasZ(eColumnGType));
1437 3892 : const bool bHasM = CPL_TO_BOOL(OGR_GT_HasM(eColumnGType));
1438 3892 : const bool bIsEmpty = poGeom != nullptr && poGeom->IsEmpty();
1439 3892 : OGREnvelope3D oEnvelope;
1440 3892 : if (poGeom != nullptr && !bIsEmpty)
1441 : {
1442 2258 : if (poGeom->Is3D())
1443 : {
1444 322 : poGeom->getEnvelope(&oEnvelope);
1445 322 : m_aoEnvelopes[iGeomField].Merge(oEnvelope);
1446 : }
1447 : else
1448 : {
1449 1936 : poGeom->getEnvelope(static_cast<OGREnvelope *>(&oEnvelope));
1450 1936 : m_aoEnvelopes[iGeomField].Merge(oEnvelope);
1451 : }
1452 2258 : m_oSetWrittenGeometryTypes[iGeomField].insert(eGType);
1453 : }
1454 :
1455 3892 : if (m_bWriteBBoxStruct)
1456 : {
1457 3185 : if (poGeom && !bIsEmpty)
1458 : {
1459 1867 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1460 : m_apoBuildersBBOXXMin[iGeomField]->Append(
1461 : castToFloatDown(oEnvelope.MinX)));
1462 1867 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1463 : m_apoBuildersBBOXYMin[iGeomField]->Append(
1464 : castToFloatDown(oEnvelope.MinY)));
1465 1867 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1466 : m_apoBuildersBBOXXMax[iGeomField]->Append(
1467 : castToFloatUp(oEnvelope.MaxX)));
1468 1867 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1469 : m_apoBuildersBBOXYMax[iGeomField]->Append(
1470 : castToFloatUp(oEnvelope.MaxY)));
1471 1867 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1472 : m_apoBuildersBBOXStruct[iGeomField]->Append());
1473 : }
1474 : else
1475 : {
1476 1318 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1477 : m_apoBuildersBBOXStruct[iGeomField]->AppendNull());
1478 : }
1479 : }
1480 :
1481 3892 : if (poGeom == nullptr)
1482 : {
1483 4068 : if (m_aeGeomEncoding[iGeomField] ==
1484 1364 : OGRArrowGeomEncoding::GEOARROW_FSL_POINT &&
1485 1364 : GetDriverUCName() == "PARQUET")
1486 : {
1487 : // For some reason, Parquet doesn't support a NULL FixedSizeList
1488 : // on reading
1489 4 : auto poPointBuilder =
1490 : static_cast<arrow::FixedSizeListBuilder *>(poBuilder);
1491 4 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1492 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1493 4 : poPointBuilder->value_builder());
1494 4 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1495 : std::numeric_limits<double>::quiet_NaN()));
1496 4 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1497 : std::numeric_limits<double>::quiet_NaN()));
1498 4 : if (bHasZ)
1499 2 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1500 : std::numeric_limits<double>::quiet_NaN()));
1501 4 : if (bHasM)
1502 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1503 : std::numeric_limits<double>::quiet_NaN()));
1504 : }
1505 : else
1506 : {
1507 1352 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1508 : }
1509 :
1510 1356 : return OGRERR_NONE;
1511 : }
1512 :
1513 : // The following checks are only valid for GeoArrow encoding
1514 3714 : if (m_aeGeomEncoding[iGeomField] != OGRArrowGeomEncoding::WKB &&
1515 1178 : m_aeGeomEncoding[iGeomField] != OGRArrowGeomEncoding::WKT)
1516 : {
1517 1114 : if ((!bIsEmpty && eGType != eColumnGType) ||
1518 244 : (bIsEmpty && wkbFlatten(eGType) != wkbFlatten(eColumnGType)))
1519 : {
1520 6 : CPLError(CE_Warning, CPLE_AppDefined,
1521 : "Geometry of type %s found, whereas %s is expected. "
1522 : "Writing null geometry",
1523 : OGRGeometryTypeToName(eGType),
1524 : OGRGeometryTypeToName(eColumnGType));
1525 6 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1526 :
1527 6 : return OGRERR_NONE;
1528 : }
1529 : }
1530 :
1531 2530 : switch (m_aeGeomEncoding[iGeomField])
1532 : {
1533 1358 : case OGRArrowGeomEncoding::WKB:
1534 : {
1535 0 : std::unique_ptr<OGRGeometry> poGeomModified;
1536 1358 : if (OGR_GT_HasM(eGType) && !OGR_GT_HasM(eColumnGType))
1537 : {
1538 : static bool bHasWarned = false;
1539 0 : if (!bHasWarned)
1540 : {
1541 0 : CPLError(CE_Warning, CPLE_AppDefined,
1542 : "Removing M component from geometry");
1543 0 : bHasWarned = true;
1544 : }
1545 0 : poGeomModified.reset(poGeom->clone());
1546 0 : poGeomModified->setMeasured(false);
1547 0 : poGeom = poGeomModified.get();
1548 : }
1549 1358 : FixupGeometryBeforeWriting(poGeom);
1550 1358 : const auto nSize = poGeom->WkbSize();
1551 1358 : if (nSize < INT_MAX)
1552 : {
1553 1358 : m_abyBuffer.resize(nSize);
1554 1358 : poGeom->exportToWkb(wkbNDR, &m_abyBuffer[0], wkbVariantIso);
1555 1358 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1556 : static_cast<arrow::BinaryBuilder *>(poBuilder)->Append(
1557 : m_abyBuffer.data(),
1558 : static_cast<int>(m_abyBuffer.size())));
1559 : }
1560 : else
1561 : {
1562 0 : CPLError(CE_Warning, CPLE_AppDefined,
1563 : "Too big geometry. "
1564 : "Writing null geometry");
1565 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1566 : }
1567 1358 : break;
1568 : }
1569 :
1570 308 : case OGRArrowGeomEncoding::WKT:
1571 : {
1572 308 : OGRWktOptions options;
1573 308 : options.variant = wkbVariantIso;
1574 308 : if (m_nWKTCoordinatePrecision >= 0)
1575 : {
1576 0 : options.format = OGRWktFormat::F;
1577 0 : options.xyPrecision = m_nWKTCoordinatePrecision;
1578 0 : options.zPrecision = m_nWKTCoordinatePrecision;
1579 0 : options.mPrecision = m_nWKTCoordinatePrecision;
1580 : }
1581 308 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1582 : static_cast<arrow::StringBuilder *>(poBuilder)->Append(
1583 : poGeom->exportToWkt(options)));
1584 308 : break;
1585 : }
1586 :
1587 20 : case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
1588 : {
1589 20 : const auto poPoint = poGeom->toPoint();
1590 20 : auto poPointBuilder =
1591 : static_cast<arrow::FixedSizeListBuilder *>(poBuilder);
1592 20 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1593 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1594 20 : poPointBuilder->value_builder());
1595 20 : if (bIsEmpty)
1596 : {
1597 8 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1598 : std::numeric_limits<double>::quiet_NaN()));
1599 8 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1600 : std::numeric_limits<double>::quiet_NaN()));
1601 8 : if (bHasZ)
1602 4 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1603 : std::numeric_limits<double>::quiet_NaN()));
1604 8 : if (bHasM)
1605 2 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1606 : std::numeric_limits<double>::quiet_NaN()));
1607 : }
1608 : else
1609 : {
1610 12 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1611 : poValueBuilder->Append(poPoint->getX()));
1612 12 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1613 : poValueBuilder->Append(poPoint->getY()));
1614 12 : if (bHasZ)
1615 6 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1616 : poValueBuilder->Append(poPoint->getZ()));
1617 12 : if (bHasM)
1618 2 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1619 : poValueBuilder->Append(poPoint->getM()));
1620 : }
1621 20 : break;
1622 : }
1623 :
1624 : #define GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder) \
1625 : auto poXBuilder = \
1626 : static_cast<arrow::DoubleBuilder *>(poPointBuilder->field_builder(0)); \
1627 : auto poYBuilder = \
1628 : static_cast<arrow::DoubleBuilder *>(poPointBuilder->field_builder(1)); \
1629 : int iSubField = 2; \
1630 : arrow::DoubleBuilder *poZBuilder = nullptr; \
1631 : if (bHasZ) \
1632 : { \
1633 : poZBuilder = static_cast<arrow::DoubleBuilder *>( \
1634 : poPointBuilder->field_builder(iSubField)); \
1635 : ++iSubField; \
1636 : } \
1637 : arrow::DoubleBuilder *poMBuilder = nullptr; \
1638 : if (bHasM) \
1639 : { \
1640 : poMBuilder = static_cast<arrow::DoubleBuilder *>( \
1641 : poPointBuilder->field_builder(iSubField)); \
1642 : } \
1643 : do \
1644 : { \
1645 : } while (0)
1646 :
1647 109 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
1648 : {
1649 109 : const auto poPoint = poGeom->toPoint();
1650 109 : auto poPointBuilder =
1651 : static_cast<arrow::StructBuilder *>(poBuilder);
1652 109 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1653 109 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1654 :
1655 109 : if (bIsEmpty)
1656 : {
1657 28 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poXBuilder->Append(
1658 : std::numeric_limits<double>::quiet_NaN()));
1659 28 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poYBuilder->Append(
1660 : std::numeric_limits<double>::quiet_NaN()));
1661 : }
1662 : else
1663 : {
1664 81 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1665 : poXBuilder->Append(poPoint->getX()));
1666 81 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1667 : poYBuilder->Append(poPoint->getY()));
1668 : }
1669 109 : if (poZBuilder)
1670 : {
1671 40 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poZBuilder->Append(
1672 : bIsEmpty ? std::numeric_limits<double>::quiet_NaN()
1673 : : poPoint->getZ()));
1674 : }
1675 109 : if (poMBuilder)
1676 : {
1677 4 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMBuilder->Append(
1678 : bIsEmpty ? std::numeric_limits<double>::quiet_NaN()
1679 : : poPoint->getM()));
1680 : }
1681 109 : break;
1682 : }
1683 :
1684 20 : case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
1685 : {
1686 20 : const auto poLS = poGeom->toLineString();
1687 20 : auto poListBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1688 : auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1689 20 : poListBuilder->value_builder());
1690 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1691 20 : poPointBuilder->value_builder());
1692 :
1693 20 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
1694 20 : OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1695 : poLS, poPointBuilder, poValueBuilder, poValueBuilder,
1696 : bHasZ ? poValueBuilder : nullptr,
1697 : bHasM ? poValueBuilder : nullptr));
1698 20 : break;
1699 : }
1700 :
1701 81 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
1702 : {
1703 81 : const auto poLS = poGeom->toLineString();
1704 81 : auto poListBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1705 : auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1706 81 : poListBuilder->value_builder());
1707 81 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1708 :
1709 81 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
1710 81 : OGR_ARROW_PROPAGATE_OGRERR(
1711 : GeoArrowLineBuilder(poLS, poPointBuilder, poXBuilder,
1712 : poYBuilder, poZBuilder, poMBuilder));
1713 81 : break;
1714 : }
1715 :
1716 32 : case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
1717 : {
1718 32 : const auto poPolygon = poGeom->toPolygon();
1719 32 : auto poPolygonBuilder =
1720 : static_cast<arrow::ListBuilder *>(poBuilder);
1721 : auto poRingBuilder = static_cast<arrow::ListBuilder *>(
1722 32 : poPolygonBuilder->value_builder());
1723 : auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1724 32 : poRingBuilder->value_builder());
1725 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1726 32 : poPointBuilder->value_builder());
1727 32 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPolygonBuilder->Append());
1728 62 : for (const auto *poRing : *poPolygon)
1729 : {
1730 30 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poRingBuilder->Append());
1731 30 : OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1732 : poRing, poPointBuilder, poValueBuilder, poValueBuilder,
1733 : bHasZ ? poValueBuilder : nullptr,
1734 : bHasM ? poValueBuilder : nullptr));
1735 : }
1736 32 : break;
1737 : }
1738 :
1739 133 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
1740 : {
1741 133 : const auto poPolygon = poGeom->toPolygon();
1742 133 : auto poPolygonBuilder =
1743 : static_cast<arrow::ListBuilder *>(poBuilder);
1744 : auto poRingBuilder = static_cast<arrow::ListBuilder *>(
1745 133 : poPolygonBuilder->value_builder());
1746 : auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1747 133 : poRingBuilder->value_builder());
1748 133 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1749 :
1750 133 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPolygonBuilder->Append());
1751 254 : for (const auto *poRing : *poPolygon)
1752 : {
1753 121 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poRingBuilder->Append());
1754 121 : OGR_ARROW_PROPAGATE_OGRERR(
1755 : GeoArrowLineBuilder(poRing, poPointBuilder, poXBuilder,
1756 : poYBuilder, poZBuilder, poMBuilder));
1757 : }
1758 133 : break;
1759 : }
1760 :
1761 32 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
1762 : {
1763 32 : const auto poMultiPoint = poGeom->toMultiPoint();
1764 32 : auto poListBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1765 : auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1766 32 : poListBuilder->value_builder());
1767 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1768 32 : poPointBuilder->value_builder());
1769 32 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
1770 88 : for (const auto *poPoint : *poMultiPoint)
1771 : {
1772 56 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1773 56 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1774 : poValueBuilder->Append(poPoint->getX()));
1775 56 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1776 : poValueBuilder->Append(poPoint->getY()));
1777 56 : if (bHasZ)
1778 28 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1779 : poValueBuilder->Append(poPoint->getZ()));
1780 56 : if (bHasM)
1781 18 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1782 : poValueBuilder->Append(poPoint->getM()));
1783 : }
1784 32 : break;
1785 : }
1786 :
1787 113 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
1788 : {
1789 113 : const auto poMultiPoint = poGeom->toMultiPoint();
1790 113 : auto poListBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1791 : auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1792 113 : poListBuilder->value_builder());
1793 113 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1794 :
1795 113 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
1796 270 : for (const auto *poPoint : *poMultiPoint)
1797 : {
1798 157 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1799 157 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1800 : poXBuilder->Append(poPoint->getX()));
1801 157 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1802 : poYBuilder->Append(poPoint->getY()));
1803 157 : if (poZBuilder)
1804 78 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1805 : poZBuilder->Append(poPoint->getZ()));
1806 157 : if (poMBuilder)
1807 18 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1808 : poMBuilder->Append(poPoint->getM()));
1809 : }
1810 113 : break;
1811 : }
1812 :
1813 28 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
1814 : {
1815 28 : const auto poMLS = poGeom->toMultiLineString();
1816 28 : auto poMLSBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1817 : auto poLSBuilder = static_cast<arrow::ListBuilder *>(
1818 28 : poMLSBuilder->value_builder());
1819 : auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1820 28 : poLSBuilder->value_builder());
1821 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1822 28 : poPointBuilder->value_builder());
1823 28 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMLSBuilder->Append());
1824 60 : for (const auto *poLS : *poMLS)
1825 : {
1826 32 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poLSBuilder->Append());
1827 32 : OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1828 : poLS, poPointBuilder, poValueBuilder, poValueBuilder,
1829 : bHasZ ? poValueBuilder : nullptr,
1830 : bHasM ? poValueBuilder : nullptr));
1831 : }
1832 28 : break;
1833 : }
1834 :
1835 109 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
1836 : {
1837 109 : const auto poMLS = poGeom->toMultiLineString();
1838 109 : auto poMLSBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1839 : auto poLSBuilder = static_cast<arrow::ListBuilder *>(
1840 109 : poMLSBuilder->value_builder());
1841 : auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1842 109 : poLSBuilder->value_builder());
1843 109 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1844 :
1845 109 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMLSBuilder->Append());
1846 242 : for (const auto *poLS : *poMLS)
1847 : {
1848 133 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poLSBuilder->Append());
1849 133 : OGR_ARROW_PROPAGATE_OGRERR(
1850 : GeoArrowLineBuilder(poLS, poPointBuilder, poXBuilder,
1851 : poYBuilder, poZBuilder, poMBuilder));
1852 : }
1853 109 : break;
1854 : }
1855 :
1856 38 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
1857 : {
1858 38 : const auto poMPoly = poGeom->toMultiPolygon();
1859 38 : auto poMPolyBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1860 : auto poPolyBuilder = static_cast<arrow::ListBuilder *>(
1861 38 : poMPolyBuilder->value_builder());
1862 : auto poRingBuilder = static_cast<arrow::ListBuilder *>(
1863 38 : poPolyBuilder->value_builder());
1864 : auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1865 38 : poRingBuilder->value_builder());
1866 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1867 38 : poPointBuilder->value_builder());
1868 38 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMPolyBuilder->Append());
1869 82 : for (const auto *poPolygon : *poMPoly)
1870 : {
1871 44 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPolyBuilder->Append());
1872 98 : for (const auto *poRing : *poPolygon)
1873 : {
1874 54 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poRingBuilder->Append());
1875 54 : OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1876 : poRing, poPointBuilder, poValueBuilder, poValueBuilder,
1877 : bHasZ ? poValueBuilder : nullptr,
1878 : bHasM ? poValueBuilder : nullptr));
1879 : }
1880 : }
1881 38 : break;
1882 : }
1883 :
1884 149 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
1885 : {
1886 149 : const auto poMPoly = poGeom->toMultiPolygon();
1887 149 : auto poMPolyBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1888 : auto poPolyBuilder = static_cast<arrow::ListBuilder *>(
1889 149 : poMPolyBuilder->value_builder());
1890 : auto poRingBuilder = static_cast<arrow::ListBuilder *>(
1891 149 : poPolyBuilder->value_builder());
1892 : auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1893 149 : poRingBuilder->value_builder());
1894 149 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1895 :
1896 149 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMPolyBuilder->Append());
1897 314 : for (const auto *poPolygon : *poMPoly)
1898 : {
1899 165 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPolyBuilder->Append());
1900 370 : for (const auto *poRing : *poPolygon)
1901 : {
1902 205 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poRingBuilder->Append());
1903 205 : OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1904 : poRing, poPointBuilder, poXBuilder, poYBuilder,
1905 : poZBuilder, poMBuilder));
1906 : }
1907 : }
1908 149 : break;
1909 : }
1910 :
1911 0 : case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
1912 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
1913 : {
1914 0 : CPLAssert(false);
1915 : break;
1916 : }
1917 : }
1918 :
1919 2530 : return OGRERR_NONE;
1920 : }
1921 :
1922 : /************************************************************************/
1923 : /* ICreateFeature() */
1924 : /************************************************************************/
1925 :
1926 3441 : inline OGRErr OGRArrowWriterLayer::ICreateFeature(OGRFeature *poFeature)
1927 : {
1928 3441 : if (m_poSchema == nullptr)
1929 : {
1930 315 : CreateSchema();
1931 : }
1932 :
1933 3441 : if (m_apoBuilders.empty())
1934 : {
1935 353 : if (!m_apoFieldsFromArrowSchema.empty())
1936 : {
1937 0 : CPLError(CE_Failure, CPLE_NotSupported,
1938 : "ICreateFeature() cannot be used after "
1939 : "CreateFieldFromArrowSchema()");
1940 0 : return OGRERR_FAILURE;
1941 : }
1942 353 : CreateArrayBuilders();
1943 : }
1944 :
1945 : // First pass to check not-null constraints as Arrow doesn't seem
1946 : // to do that on the writing side. But such files can't be read.
1947 3441 : const int nFieldCount = m_poFeatureDefn->GetFieldCount();
1948 9048 : for (int i = 0; i < nFieldCount; ++i)
1949 : {
1950 5608 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
1951 5610 : if (!poFieldDefn->IsNullable() &&
1952 2 : !poFeature->IsFieldSetAndNotNullUnsafe(i))
1953 : {
1954 1 : CPLError(CE_Failure, CPLE_AppDefined,
1955 : "Null value found in non-nullable field %s",
1956 : poFieldDefn->GetNameRef());
1957 1 : return OGRERR_FAILURE;
1958 : }
1959 : }
1960 :
1961 3440 : const int nGeomFieldCount = m_poFeatureDefn->GetGeomFieldCount();
1962 6983 : for (int i = 0; i < nGeomFieldCount; ++i)
1963 : {
1964 3543 : const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
1965 3575 : if (!poGeomFieldDefn->IsNullable() &&
1966 32 : poFeature->GetGeomFieldRef(i) == nullptr)
1967 : {
1968 0 : CPLError(CE_Failure, CPLE_AppDefined,
1969 : "Null value found in non-nullable geometry field %s",
1970 : poGeomFieldDefn->GetNameRef());
1971 0 : return OGRERR_FAILURE;
1972 : }
1973 : }
1974 :
1975 : // Write FID, if FID column present
1976 3440 : int nArrowIdx = 0;
1977 3440 : if (!m_osFIDColumn.empty())
1978 : {
1979 2261 : int64_t nFID = poFeature->GetFID();
1980 2261 : if (nFID == OGRNullFID)
1981 : {
1982 37 : nFID = m_nFeatureCount;
1983 37 : poFeature->SetFID(nFID);
1984 : }
1985 : auto poBuilder =
1986 2261 : static_cast<arrow::Int64Builder *>(m_apoBuilders[0].get());
1987 2261 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->Append(nFID));
1988 2261 : nArrowIdx++;
1989 : }
1990 :
1991 : // Write attributes
1992 9047 : for (int i = 0; i < nFieldCount; ++i, ++nArrowIdx)
1993 : {
1994 5607 : auto poBuilder = m_apoBuilders[nArrowIdx].get();
1995 5607 : if (!poFeature->IsFieldSetAndNotNullUnsafe(i))
1996 : {
1997 1248 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1998 1248 : continue;
1999 : }
2000 :
2001 4359 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
2002 4359 : const auto eSubDT = poFieldDefn->GetSubType();
2003 4359 : switch (poFieldDefn->GetType())
2004 : {
2005 2485 : case OFTInteger:
2006 2485 : if (eSubDT == OFSTBoolean)
2007 16 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2008 : static_cast<arrow::BooleanBuilder *>(poBuilder)->Append(
2009 : poFeature->GetFieldAsIntegerUnsafe(i) != 0));
2010 2469 : else if (eSubDT == OFSTInt16)
2011 16 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2012 : static_cast<arrow::Int16Builder *>(poBuilder)->Append(
2013 : static_cast<int16_t>(
2014 : poFeature->GetFieldAsIntegerUnsafe(i))));
2015 : else
2016 2453 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2017 : static_cast<arrow::Int32Builder *>(poBuilder)->Append(
2018 : poFeature->GetFieldAsIntegerUnsafe(i)));
2019 2485 : break;
2020 :
2021 172 : case OFTInteger64:
2022 172 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2023 : static_cast<arrow::Int64Builder *>(poBuilder)->Append(
2024 : static_cast<int64_t>(
2025 : poFeature->GetFieldAsInteger64Unsafe(i))));
2026 172 : break;
2027 :
2028 236 : case OFTReal:
2029 : {
2030 236 : const auto arrowType = m_poSchema->fields()[nArrowIdx]->type();
2031 236 : const double dfVal = poFeature->GetFieldAsDoubleUnsafe(i);
2032 236 : if (arrowType->id() == arrow::Type::DECIMAL128)
2033 : {
2034 : auto res = arrow::Decimal128::FromReal(
2035 : dfVal, poFieldDefn->GetWidth(),
2036 62 : poFieldDefn->GetPrecision());
2037 62 : if (res.ok())
2038 : {
2039 62 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2040 : static_cast<arrow::Decimal128Builder *>(poBuilder)
2041 : ->Append(*res));
2042 : }
2043 : else
2044 : {
2045 0 : CPLError(CE_Warning, CPLE_AppDefined,
2046 : "Cannot parse %.18g as a %d.%d decimal", dfVal,
2047 : poFieldDefn->GetWidth(),
2048 : poFieldDefn->GetPrecision());
2049 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
2050 : }
2051 : }
2052 174 : else if (arrowType->id() == arrow::Type::DECIMAL256)
2053 : {
2054 : auto res = arrow::Decimal256::FromReal(
2055 : dfVal, poFieldDefn->GetWidth(),
2056 0 : poFieldDefn->GetPrecision());
2057 0 : if (res.ok())
2058 : {
2059 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2060 : static_cast<arrow::Decimal256Builder *>(poBuilder)
2061 : ->Append(*res));
2062 : }
2063 : else
2064 : {
2065 0 : CPLError(CE_Warning, CPLE_AppDefined,
2066 : "Cannot parse %.18g as a %d.%d decimal", dfVal,
2067 : poFieldDefn->GetWidth(),
2068 : poFieldDefn->GetPrecision());
2069 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
2070 : }
2071 : }
2072 174 : else if (eSubDT == OFSTFloat32)
2073 : {
2074 28 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2075 : static_cast<arrow::FloatBuilder *>(poBuilder)->Append(
2076 : static_cast<float>(dfVal)));
2077 : }
2078 : else
2079 : {
2080 146 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2081 : static_cast<arrow::DoubleBuilder *>(poBuilder)->Append(
2082 : dfVal));
2083 : }
2084 236 : break;
2085 : }
2086 :
2087 545 : case OFTString:
2088 : case OFTWideString:
2089 545 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2090 : static_cast<arrow::StringBuilder *>(poBuilder)->Append(
2091 : poFeature->GetFieldAsStringUnsafe(i)));
2092 545 : break;
2093 :
2094 62 : case OFTBinary:
2095 : {
2096 62 : int nSize = 0;
2097 62 : const auto pData = poFeature->GetFieldAsBinary(i, &nSize);
2098 62 : if (poFieldDefn->GetWidth() != 0)
2099 : {
2100 20 : if (poFieldDefn->GetWidth() != nSize)
2101 : {
2102 0 : CPLError(
2103 : CE_Warning, CPLE_AppDefined,
2104 : "Cannot write field %s. Got %d bytes, expected %d",
2105 : poFieldDefn->GetNameRef(), nSize,
2106 : poFieldDefn->GetWidth());
2107 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
2108 : }
2109 : else
2110 : {
2111 20 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2112 : static_cast<arrow::FixedSizeBinaryBuilder *>(
2113 : poBuilder)
2114 : ->Append(pData));
2115 : }
2116 : }
2117 : else
2118 42 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2119 : static_cast<arrow::BinaryBuilder *>(poBuilder)->Append(
2120 : pData, nSize));
2121 62 : break;
2122 : }
2123 :
2124 216 : case OFTIntegerList:
2125 : {
2126 216 : auto poListBuilder =
2127 : static_cast<arrow::ListBuilder *>(poBuilder);
2128 216 : if (eSubDT == OFSTBoolean)
2129 : {
2130 36 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2131 : auto poValueBuilder = static_cast<arrow::BooleanBuilder *>(
2132 36 : poListBuilder->value_builder());
2133 36 : int nValues = 0;
2134 : const auto panValues =
2135 36 : poFeature->GetFieldAsIntegerList(i, &nValues);
2136 108 : for (int j = 0; j < nValues; ++j)
2137 72 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2138 : poValueBuilder->Append(panValues[j] != 0));
2139 : }
2140 180 : else if (eSubDT == OFSTInt16)
2141 : {
2142 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2143 : auto poValueBuilder = static_cast<arrow::Int16Builder *>(
2144 0 : poListBuilder->value_builder());
2145 0 : int nValues = 0;
2146 : const auto panValues =
2147 0 : poFeature->GetFieldAsIntegerList(i, &nValues);
2148 0 : for (int j = 0; j < nValues; ++j)
2149 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
2150 : static_cast<int16_t>(panValues[j])));
2151 : }
2152 : else
2153 : {
2154 180 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2155 : auto poValueBuilder = static_cast<arrow::Int32Builder *>(
2156 180 : poListBuilder->value_builder());
2157 180 : int nValues = 0;
2158 : const auto panValues =
2159 180 : poFeature->GetFieldAsIntegerList(i, &nValues);
2160 540 : for (int j = 0; j < nValues; ++j)
2161 360 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2162 : poValueBuilder->Append(panValues[j]));
2163 : }
2164 216 : break;
2165 : }
2166 :
2167 92 : case OFTInteger64List:
2168 : {
2169 92 : auto poListBuilder =
2170 : static_cast<arrow::ListBuilder *>(poBuilder);
2171 92 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2172 : auto poValueBuilder = static_cast<arrow::Int64Builder *>(
2173 92 : poListBuilder->value_builder());
2174 92 : int nValues = 0;
2175 : const auto panValues =
2176 92 : poFeature->GetFieldAsInteger64List(i, &nValues);
2177 292 : for (int j = 0; j < nValues; ++j)
2178 200 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
2179 : static_cast<int64_t>(panValues[j])));
2180 92 : break;
2181 : }
2182 :
2183 152 : case OFTRealList:
2184 : {
2185 152 : auto poListBuilder =
2186 : static_cast<arrow::ListBuilder *>(poBuilder);
2187 152 : if (eSubDT == OFSTFloat32)
2188 : {
2189 48 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2190 : auto poValueBuilder = static_cast<arrow::FloatBuilder *>(
2191 48 : poListBuilder->value_builder());
2192 48 : int nValues = 0;
2193 : const auto padfValues =
2194 48 : poFeature->GetFieldAsDoubleList(i, &nValues);
2195 144 : for (int j = 0; j < nValues; ++j)
2196 96 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
2197 : static_cast<float>(padfValues[j])));
2198 : }
2199 : else
2200 : {
2201 104 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2202 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
2203 104 : poListBuilder->value_builder());
2204 104 : int nValues = 0;
2205 : const auto padfValues =
2206 104 : poFeature->GetFieldAsDoubleList(i, &nValues);
2207 280 : for (int j = 0; j < nValues; ++j)
2208 176 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2209 : poValueBuilder->Append(padfValues[j]));
2210 : }
2211 152 : break;
2212 : }
2213 :
2214 52 : case OFTStringList:
2215 : case OFTWideStringList:
2216 : {
2217 52 : auto poListBuilder =
2218 : static_cast<arrow::ListBuilder *>(poBuilder);
2219 52 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2220 : auto poValueBuilder = static_cast<arrow::StringBuilder *>(
2221 52 : poListBuilder->value_builder());
2222 52 : const auto papszValues = poFeature->GetFieldAsStringList(i);
2223 132 : for (int j = 0; papszValues && papszValues[j]; ++j)
2224 80 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2225 : poValueBuilder->Append(papszValues[j]));
2226 52 : break;
2227 : }
2228 :
2229 109 : case OFTDate:
2230 : {
2231 : int nYear, nMonth, nDay, nHour, nMinute;
2232 : float fSec;
2233 : int nTZFlag;
2234 109 : poFeature->GetFieldAsDateTime(i, &nYear, &nMonth, &nDay, &nHour,
2235 : &nMinute, &fSec, &nTZFlag);
2236 : struct tm brokenDown;
2237 109 : memset(&brokenDown, 0, sizeof(brokenDown));
2238 109 : brokenDown.tm_year = nYear - 1900;
2239 109 : brokenDown.tm_mon = nMonth - 1;
2240 109 : brokenDown.tm_mday = nDay;
2241 109 : GIntBig nVal = CPLYMDHMSToUnixTime(&brokenDown);
2242 109 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2243 : static_cast<arrow::Date32Builder *>(poBuilder)->Append(
2244 : static_cast<int>(nVal / 86400)));
2245 109 : break;
2246 : }
2247 :
2248 36 : case OFTTime:
2249 : {
2250 : int nYear, nMonth, nDay, nHour, nMinute;
2251 : float fSec;
2252 : int nTZFlag;
2253 36 : poFeature->GetFieldAsDateTime(i, &nYear, &nMonth, &nDay, &nHour,
2254 : &nMinute, &fSec, &nTZFlag);
2255 36 : int nVal = nHour * 3600 + nMinute * 60;
2256 36 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2257 : static_cast<arrow::Time32Builder *>(poBuilder)->Append(
2258 : static_cast<int>(
2259 : (static_cast<double>(nVal) + fSec) * 1000 + 0.5)));
2260 36 : break;
2261 : }
2262 :
2263 202 : case OFTDateTime:
2264 : {
2265 : int nYear, nMonth, nDay, nHour, nMinute;
2266 : float fSec;
2267 : int nTZFlag;
2268 202 : poFeature->GetFieldAsDateTime(i, &nYear, &nMonth, &nDay, &nHour,
2269 : &nMinute, &fSec, &nTZFlag);
2270 : struct tm brokenDown;
2271 202 : memset(&brokenDown, 0, sizeof(brokenDown));
2272 202 : brokenDown.tm_year = nYear - 1900;
2273 202 : brokenDown.tm_mon = nMonth - 1;
2274 202 : brokenDown.tm_mday = nDay;
2275 202 : brokenDown.tm_hour = nHour;
2276 202 : brokenDown.tm_min = nMinute;
2277 202 : brokenDown.tm_sec = 0;
2278 202 : GIntBig nVal = CPLYMDHMSToUnixTime(&brokenDown);
2279 332 : if (!IsFileWriterCreated() &&
2280 130 : m_anTZFlag[i] != OGR_TZFLAG_UNKNOWN)
2281 : {
2282 72 : if (m_anTZFlag[i] == TZFLAG_UNINITIALIZED)
2283 35 : m_anTZFlag[i] = nTZFlag;
2284 37 : else if (m_anTZFlag[i] != nTZFlag)
2285 : {
2286 26 : if (m_anTZFlag[i] >= OGR_TZFLAG_MIXED_TZ &&
2287 13 : nTZFlag >= OGR_TZFLAG_MIXED_TZ)
2288 : {
2289 10 : m_anTZFlag[i] =
2290 : OGR_TZFLAG_MIXED_TZ; // harmonize on UTC ultimately
2291 : }
2292 : else
2293 : {
2294 3 : CPLError(CE_Warning, CPLE_AppDefined,
2295 : "Field %s contains a mix of "
2296 : "timezone-aware and local/without "
2297 : "timezone values.",
2298 : poFieldDefn->GetNameRef());
2299 3 : m_anTZFlag[i] = OGR_TZFLAG_UNKNOWN;
2300 : }
2301 : }
2302 : }
2303 202 : if (nTZFlag > OGR_TZFLAG_MIXED_TZ)
2304 : {
2305 70 : const int nOffsetSec = (nTZFlag - OGR_TZFLAG_UTC) * 15 * 60;
2306 70 : nVal -= nOffsetSec;
2307 : }
2308 202 : const int64_t nTimestamp = static_cast<int64_t>(
2309 202 : (static_cast<double>(nVal) + fSec) * 1000 + 0.5);
2310 : auto structBuilder =
2311 202 : dynamic_cast<arrow::StructBuilder *>(poBuilder);
2312 202 : if (structBuilder)
2313 : {
2314 10 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2315 : static_cast<arrow::TimestampBuilder *>(
2316 : structBuilder->field_builder(0))
2317 : ->Append(nTimestamp));
2318 10 : const int16_t nUTCOffsetMin =
2319 10 : static_cast<int16_t>((nTZFlag - OGR_TZFLAG_UTC) * 15);
2320 10 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2321 : static_cast<arrow::Int16Builder *>(
2322 : structBuilder->field_builder(1))
2323 : ->Append(nUTCOffsetMin));
2324 10 : OGR_ARROW_RETURN_OGRERR_NOT_OK(structBuilder->Append());
2325 : }
2326 : else
2327 : {
2328 192 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2329 : static_cast<arrow::TimestampBuilder *>(poBuilder)
2330 : ->Append(nTimestamp));
2331 : }
2332 202 : break;
2333 : }
2334 : }
2335 : }
2336 :
2337 : // Write geometries
2338 6983 : for (int i = 0; i < nGeomFieldCount; ++i, ++nArrowIdx)
2339 : {
2340 3543 : auto poBuilder = m_apoBuilders[nArrowIdx].get();
2341 3543 : OGRGeometry *poGeom = poFeature->GetGeomFieldRef(i);
2342 3543 : if (BuildGeometry(poGeom, i, poBuilder) != OGRERR_NONE)
2343 0 : return OGRERR_FAILURE;
2344 : }
2345 :
2346 3440 : m_nFeatureCount++;
2347 :
2348 : // Flush the current row group if reaching the limit of rows per group.
2349 3440 : if (!m_apoBuilders.empty() && m_apoBuilders[0]->length() == m_nRowGroupSize)
2350 : {
2351 25 : if (!FlushFeatures())
2352 0 : return OGRERR_FAILURE;
2353 : }
2354 :
2355 3440 : return OGRERR_NONE;
2356 : }
2357 :
2358 : /************************************************************************/
2359 : /* FlushFeatures() */
2360 : /************************************************************************/
2361 :
2362 42 : inline bool OGRArrowWriterLayer::FlushFeatures()
2363 : {
2364 42 : if (m_apoBuilders.empty() || m_apoBuilders[0]->length() == 0)
2365 1 : return true;
2366 :
2367 41 : if (!IsFileWriterCreated())
2368 : {
2369 10 : CreateWriter();
2370 10 : if (!IsFileWriterCreated())
2371 0 : return false;
2372 : }
2373 :
2374 41 : return FlushGroup();
2375 : }
2376 :
2377 : /************************************************************************/
2378 : /* GetFeatureCount() */
2379 : /************************************************************************/
2380 :
2381 1 : inline GIntBig OGRArrowWriterLayer::GetFeatureCount(int bForce)
2382 : {
2383 1 : if (m_poAttrQuery == nullptr && m_poFilterGeom == nullptr)
2384 : {
2385 1 : return m_nFeatureCount;
2386 : }
2387 0 : return OGRLayer::GetFeatureCount(bForce);
2388 : }
2389 :
2390 : /************************************************************************/
2391 : /* TestCapability() */
2392 : /************************************************************************/
2393 :
2394 1075 : inline bool OGRArrowWriterLayer::TestCapability(const char *pszCap) const
2395 : {
2396 1075 : if (EQUAL(pszCap, OLCCreateField) || EQUAL(pszCap, OLCCreateGeomField))
2397 27 : return m_poSchema == nullptr;
2398 :
2399 1048 : if (EQUAL(pszCap, OLCSequentialWrite))
2400 24 : return true;
2401 :
2402 1024 : if (EQUAL(pszCap, OLCFastWriteArrowBatch))
2403 0 : return true;
2404 :
2405 1024 : if (EQUAL(pszCap, OLCStringsAsUTF8))
2406 1 : return true;
2407 :
2408 1023 : if (EQUAL(pszCap, OLCMeasuredGeometries))
2409 446 : return true;
2410 :
2411 577 : return false;
2412 : }
2413 :
2414 : /************************************************************************/
2415 : /* WriteArrays() */
2416 : /************************************************************************/
2417 :
2418 355 : inline bool OGRArrowWriterLayer::WriteArrays(
2419 : std::function<bool(const std::shared_ptr<arrow::Field> &,
2420 : const std::shared_ptr<arrow::Array> &)>
2421 : postProcessArray)
2422 : {
2423 355 : int nArrowIdx = 0;
2424 355 : int nArrowIdxFirstField = !m_osFIDColumn.empty() ? 1 : 0;
2425 2233 : for (const auto &poBuilder : m_apoBuilders)
2426 : {
2427 1878 : const auto &field = m_poSchema->fields()[nArrowIdx];
2428 :
2429 0 : std::shared_ptr<arrow::Array> array;
2430 1878 : auto status = poBuilder->Finish(&array);
2431 1878 : if (!status.ok())
2432 : {
2433 0 : CPLError(CE_Failure, CPLE_AppDefined,
2434 : "builder::Finish() for field %s failed with %s",
2435 0 : field->name().c_str(), status.message().c_str());
2436 0 : return false;
2437 : }
2438 :
2439 : // CPLDebug("ARROW", "%s", array->ToString().c_str());
2440 :
2441 1878 : const int iCol = nArrowIdx - nArrowIdxFirstField;
2442 1878 : if (iCol >= 0 && iCol < m_poFeatureDefn->GetFieldCount())
2443 : {
2444 1482 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(iCol);
2445 1482 : const auto eFieldType = poFieldDefn->GetType();
2446 1482 : if (eFieldType == OFTInteger || eFieldType == OFTInteger64)
2447 : {
2448 261 : const auto &osDomainName = poFieldDefn->GetDomainName();
2449 : const auto oIter =
2450 261 : m_oMapFieldDomainToStringArray.find(osDomainName);
2451 261 : if (oIter != m_oMapFieldDomainToStringArray.end())
2452 : {
2453 : auto result = arrow::DictionaryArray::FromArrays(
2454 12 : field->type(), array, oIter->second);
2455 12 : if (!result.ok())
2456 : {
2457 0 : CPLError(CE_Failure, CPLE_AppDefined,
2458 : "DictionaryArray::FromArrays() for field %s "
2459 : "failed with %s",
2460 0 : field->name().c_str(),
2461 0 : result.status().message().c_str());
2462 0 : return false;
2463 : }
2464 12 : array = *result;
2465 : }
2466 : }
2467 : }
2468 :
2469 1878 : if (!postProcessArray(field, array))
2470 : {
2471 0 : return false;
2472 : }
2473 :
2474 1878 : nArrowIdx++;
2475 : }
2476 :
2477 355 : if (m_bWriteBBoxStruct)
2478 : {
2479 263 : const int nGeomFieldCount = m_poFeatureDefn->GetGeomFieldCount();
2480 522 : for (int i = 0; i < nGeomFieldCount; ++i)
2481 : {
2482 259 : const auto &field = m_apoFieldsBBOX[i];
2483 0 : std::shared_ptr<arrow::Array> array;
2484 259 : auto status = m_apoBuildersBBOXStruct[i]->Finish(&array);
2485 259 : if (!status.ok())
2486 : {
2487 0 : CPLError(CE_Failure, CPLE_AppDefined,
2488 : "builder::Finish() for field %s failed with %s",
2489 0 : field->name().c_str(), status.message().c_str());
2490 0 : return false;
2491 : }
2492 :
2493 259 : if (!postProcessArray(field, array))
2494 : {
2495 0 : return false;
2496 : }
2497 : }
2498 : }
2499 :
2500 355 : return true;
2501 : }
2502 :
2503 : /************************************************************************/
2504 : /* TestBit() */
2505 : /************************************************************************/
2506 :
2507 516 : static inline bool TestBit(const uint8_t *pabyData, size_t nIdx)
2508 : {
2509 516 : return (pabyData[nIdx / 8] & (1 << (nIdx % 8))) != 0;
2510 : }
2511 :
2512 : /************************************************************************/
2513 : /* WriteArrowBatchInternal() */
2514 : /************************************************************************/
2515 :
2516 137 : inline bool OGRArrowWriterLayer::WriteArrowBatchInternal(
2517 : const struct ArrowSchema *schema, struct ArrowArray *array,
2518 : CSLConstList papszOptions,
2519 : std::function<bool(const std::shared_ptr<arrow::RecordBatch> &)> writeBatch)
2520 : {
2521 : #ifdef __COVERITY__
2522 : (void)schema;
2523 : (void)array;
2524 : (void)papszOptions;
2525 : (void)writeBatch;
2526 : CPLError(CE_Failure, CPLE_AppDefined, "Not implemented");
2527 : return false;
2528 : #else
2529 137 : if (m_poSchema == nullptr)
2530 : {
2531 131 : CreateSchema();
2532 : }
2533 :
2534 137 : if (!IsFileWriterCreated())
2535 : {
2536 131 : CreateWriter();
2537 131 : if (!IsFileWriterCreated())
2538 0 : return false;
2539 : }
2540 :
2541 137 : if (m_apoBuilders.empty())
2542 : {
2543 131 : CreateArrayBuilders();
2544 : }
2545 :
2546 137 : const int nGeomFieldCount = m_poFeatureDefn->GetGeomFieldCount();
2547 137 : const int nGeomFieldCountBBoxFields =
2548 137 : m_bWriteBBoxStruct ? nGeomFieldCount : 0;
2549 :
2550 137 : const char *pszFIDName = CSLFetchNameValueDef(
2551 : papszOptions, "FID", OGRLayer::DEFAULT_ARROW_FID_NAME);
2552 : const char *pszSingleGeomFieldName =
2553 137 : CSLFetchNameValue(papszOptions, "GEOMETRY_NAME");
2554 :
2555 : // Sort schema and array children in the same order as m_poSchema.
2556 : // This is needed for non-WKB geometry encoding
2557 274 : std::map<std::string, int> oMapSchemaChildrenNameToIdx;
2558 1760 : for (int i = 0; i < static_cast<int>(schema->n_children); ++i)
2559 : {
2560 1623 : if (cpl::contains(oMapSchemaChildrenNameToIdx,
2561 1623 : schema->children[i]->name))
2562 : {
2563 0 : CPLError(CE_Failure, CPLE_AppDefined,
2564 : "Several fields with same name '%s' found",
2565 0 : schema->children[i]->name);
2566 0 : return false;
2567 : }
2568 1623 : oMapSchemaChildrenNameToIdx[schema->children[i]->name] = i;
2569 :
2570 1623 : if (!pszSingleGeomFieldName && schema->children[i]->metadata)
2571 : {
2572 : const auto oMetadata =
2573 284 : OGRParseArrowMetadata(schema->children[i]->metadata);
2574 142 : const auto oIter = oMetadata.find(ARROW_EXTENSION_NAME_KEY);
2575 301 : if (oIter != oMetadata.end() &&
2576 159 : (oIter->second == EXTENSION_NAME_OGC_WKB ||
2577 19 : oIter->second == EXTENSION_NAME_GEOARROW_WKB))
2578 : {
2579 129 : pszSingleGeomFieldName = schema->children[i]->name;
2580 : }
2581 : }
2582 : }
2583 137 : if (!pszSingleGeomFieldName)
2584 8 : pszSingleGeomFieldName = OGRLayer::DEFAULT_ARROW_GEOMETRY_NAME;
2585 :
2586 137 : std::vector<int> anMapLayerSchemaToArraySchema(m_poSchema->num_fields(),
2587 411 : -1);
2588 : struct ArrowArray fidArray;
2589 : struct ArrowSchema fidSchema;
2590 137 : memset(&fidArray, 0, sizeof(fidArray));
2591 137 : memset(&fidSchema, 0, sizeof(fidSchema));
2592 274 : std::vector<void *> apBuffersFid;
2593 274 : std::vector<int64_t> fids;
2594 :
2595 274 : std::set<int> oSetReferencedFieldsInArraySchema;
2596 0 : const auto DummyFreeArray = [](struct ArrowArray *ptrArray)
2597 0 : { ptrArray->release = nullptr; };
2598 137 : const auto DummyFreeSchema = [](struct ArrowSchema *ptrSchema)
2599 137 : { ptrSchema->release = nullptr; };
2600 137 : bool bRebuildBatch = false;
2601 1758 : for (int i = 0; i < m_poSchema->num_fields() - nGeomFieldCountBBoxFields;
2602 : ++i)
2603 : {
2604 : auto oIter =
2605 1621 : oMapSchemaChildrenNameToIdx.find(m_poSchema->field(i)->name());
2606 1621 : if (oIter == oMapSchemaChildrenNameToIdx.end())
2607 : {
2608 7 : if (m_poSchema->field(i)->name() == m_osFIDColumn)
2609 : {
2610 1 : oIter = oMapSchemaChildrenNameToIdx.find(pszFIDName);
2611 1 : if (oIter == oMapSchemaChildrenNameToIdx.end())
2612 : {
2613 : // If the input data does not contain a FID column, but
2614 : // the output file requires it, creates a default FID column
2615 0 : fidArray.release = DummyFreeArray;
2616 0 : fidArray.n_buffers = 2;
2617 0 : apBuffersFid.resize(2);
2618 0 : fidArray.buffers =
2619 0 : const_cast<const void **>(apBuffersFid.data());
2620 0 : fids.reserve(static_cast<size_t>(array->length));
2621 0 : for (size_t iRow = 0;
2622 0 : iRow < static_cast<size_t>(array->length); ++iRow)
2623 0 : fids.push_back(m_nFeatureCount + iRow);
2624 0 : fidArray.buffers[1] = fids.data();
2625 0 : fidArray.length = array->length;
2626 0 : fidSchema.release = DummyFreeSchema;
2627 0 : fidSchema.name = m_osFIDColumn.c_str();
2628 0 : fidSchema.format = "l"; // int64
2629 0 : continue;
2630 : }
2631 : }
2632 12 : else if (nGeomFieldCount == 1 &&
2633 6 : m_poFeatureDefn->GetGeomFieldIndex(
2634 6 : m_poSchema->field(i)->name().c_str()) == 0)
2635 : {
2636 : oIter =
2637 6 : oMapSchemaChildrenNameToIdx.find(pszSingleGeomFieldName);
2638 6 : if (oIter != oMapSchemaChildrenNameToIdx.end())
2639 6 : bRebuildBatch = true;
2640 : }
2641 :
2642 7 : if (oIter == oMapSchemaChildrenNameToIdx.end())
2643 : {
2644 0 : CPLError(CE_Failure, CPLE_AppDefined,
2645 : "Cannot find field '%s' in schema",
2646 0 : m_poSchema->field(i)->name().c_str());
2647 0 : return false;
2648 : }
2649 : }
2650 1621 : anMapLayerSchemaToArraySchema[i] = oIter->second;
2651 1621 : oSetReferencedFieldsInArraySchema.insert(oIter->second);
2652 : }
2653 :
2654 : // Note: we cheat a bit by declaring a single instance of the minx/miny/
2655 : // maxx/maxy sub-field ArrowSchema*, and make all struct ArrowSchema point
2656 : // to them. That's OK because we use DummyFreeSchema to release, which does
2657 : // nothing.
2658 : struct ArrowSchema bboxStructSchemaXMin;
2659 : struct ArrowSchema bboxStructSchemaYMin;
2660 : struct ArrowSchema bboxStructSchemaXMax;
2661 : struct ArrowSchema bboxStructSchemaYMax;
2662 137 : constexpr int BBOX_SUBFIELD_COUNT = 4;
2663 : std::array<struct ArrowSchema *, BBOX_SUBFIELD_COUNT>
2664 : bboxStructSchemaChildren;
2665 137 : constexpr int BBOX_STRUCT_BUFFER_COUNT = 1; // validity bitmap array
2666 : // cppcheck-suppress constStatement
2667 : std::vector<std::array<const void *, BBOX_STRUCT_BUFFER_COUNT>>
2668 274 : bboxStructBuffersPtr;
2669 274 : std::vector<std::vector<GByte>> aabyBboxStructValidity;
2670 274 : std::vector<std::vector<float>> aadfMinX, aadfMinY, aadfMaxX, aadfMaxY;
2671 : // cppcheck-suppress constStatement
2672 274 : std::vector<std::array<struct ArrowArray, BBOX_SUBFIELD_COUNT>> bboxArrays;
2673 : // cppcheck-suppress constStatement
2674 : std::vector<std::array<struct ArrowArray *, BBOX_SUBFIELD_COUNT>>
2675 274 : bboxArraysPtr;
2676 137 : constexpr int BBOX_SUBFIELD_BUFFER_COUNT =
2677 : 2; // validity bitmap array and float array
2678 : std::vector<std::array<std::array<const void *, BBOX_SUBFIELD_BUFFER_COUNT>,
2679 : BBOX_SUBFIELD_COUNT>>
2680 274 : bboxBuffersPtr;
2681 :
2682 : // Temporary arrays to hold the geometry bounding boxes.
2683 274 : std::vector<struct ArrowArray> bboxStructArray;
2684 274 : std::vector<struct ArrowSchema> bboxStructSchema;
2685 :
2686 274 : std::vector<struct ArrowSchema *> newSchemaChildren;
2687 274 : std::vector<struct ArrowArray *> newArrayChildren;
2688 137 : newSchemaChildren.reserve(m_poSchema->num_fields());
2689 137 : newArrayChildren.reserve(m_poSchema->num_fields());
2690 1758 : for (int i = 0; i < m_poSchema->num_fields() - nGeomFieldCountBBoxFields;
2691 : ++i)
2692 : {
2693 1621 : if (anMapLayerSchemaToArraySchema[i] < 0)
2694 : {
2695 0 : CPLAssert(m_poSchema->field(i)->name() == m_osFIDColumn);
2696 0 : newSchemaChildren.emplace_back(&fidSchema);
2697 0 : newArrayChildren.emplace_back(&fidArray);
2698 : }
2699 : else
2700 : {
2701 : newSchemaChildren.emplace_back(
2702 1621 : schema->children[anMapLayerSchemaToArraySchema[i]]);
2703 : newArrayChildren.emplace_back(
2704 1621 : array->children[anMapLayerSchemaToArraySchema[i]]);
2705 : }
2706 : }
2707 :
2708 137 : if (m_bWriteBBoxStruct)
2709 : {
2710 25 : memset(&bboxStructSchemaXMin, 0, sizeof(bboxStructSchemaXMin));
2711 25 : memset(&bboxStructSchemaYMin, 0, sizeof(bboxStructSchemaYMin));
2712 25 : memset(&bboxStructSchemaXMax, 0, sizeof(bboxStructSchemaXMax));
2713 25 : memset(&bboxStructSchemaYMax, 0, sizeof(bboxStructSchemaYMax));
2714 :
2715 25 : bboxStructSchemaXMin.release = DummyFreeSchema;
2716 25 : bboxStructSchemaXMin.name = "xmin";
2717 25 : bboxStructSchemaXMin.format = "f"; // float32
2718 :
2719 25 : bboxStructSchemaYMin.release = DummyFreeSchema;
2720 25 : bboxStructSchemaYMin.name = "ymin";
2721 25 : bboxStructSchemaYMin.format = "f"; // float32
2722 :
2723 25 : bboxStructSchemaXMax.release = DummyFreeSchema;
2724 25 : bboxStructSchemaXMax.name = "xmax";
2725 25 : bboxStructSchemaXMax.format = "f"; // float32
2726 :
2727 25 : bboxStructSchemaYMax.release = DummyFreeSchema;
2728 25 : bboxStructSchemaYMax.name = "ymax";
2729 25 : bboxStructSchemaYMax.format = "f"; // float32
2730 :
2731 : try
2732 : {
2733 25 : constexpr int XMIN_IDX = 0;
2734 25 : constexpr int YMIN_IDX = 1;
2735 25 : constexpr int XMAX_IDX = 2;
2736 25 : constexpr int YMAX_IDX = 3;
2737 25 : bboxStructSchemaChildren[XMIN_IDX] = &bboxStructSchemaXMin;
2738 : // cppcheck-suppress objectIndex
2739 25 : bboxStructSchemaChildren[YMIN_IDX] = &bboxStructSchemaYMin;
2740 : // cppcheck-suppress objectIndex
2741 25 : bboxStructSchemaChildren[XMAX_IDX] = &bboxStructSchemaXMax;
2742 : // cppcheck-suppress objectIndex
2743 25 : bboxStructSchemaChildren[YMAX_IDX] = &bboxStructSchemaYMax;
2744 :
2745 25 : bboxStructArray.resize(nGeomFieldCount);
2746 25 : bboxStructSchema.resize(nGeomFieldCount);
2747 25 : bboxArrays.resize(nGeomFieldCount);
2748 25 : bboxArraysPtr.resize(nGeomFieldCount);
2749 25 : bboxBuffersPtr.resize(nGeomFieldCount);
2750 25 : bboxStructBuffersPtr.resize(nGeomFieldCount);
2751 25 : aabyBboxStructValidity.resize(nGeomFieldCount);
2752 50 : memset(bboxStructArray.data(), 0,
2753 25 : nGeomFieldCount * sizeof(bboxStructArray[0]));
2754 50 : memset(bboxStructSchema.data(), 0,
2755 25 : nGeomFieldCount * sizeof(bboxStructSchema[0]));
2756 50 : memset(bboxArrays.data(), 0,
2757 25 : nGeomFieldCount * sizeof(bboxArrays[0]));
2758 25 : aadfMinX.resize(nGeomFieldCount);
2759 25 : aadfMinY.resize(nGeomFieldCount);
2760 25 : aadfMaxX.resize(nGeomFieldCount);
2761 25 : aadfMaxY.resize(nGeomFieldCount);
2762 50 : for (int i = 0; i < nGeomFieldCount; ++i)
2763 : {
2764 25 : const bool bIsNullable = CPL_TO_BOOL(
2765 25 : m_poFeatureDefn->GetGeomFieldDefn(i)->IsNullable());
2766 25 : aadfMinX[i].reserve(static_cast<size_t>(array->length));
2767 25 : aadfMinY[i].reserve(static_cast<size_t>(array->length));
2768 25 : aadfMaxX[i].reserve(static_cast<size_t>(array->length));
2769 25 : aadfMaxY[i].reserve(static_cast<size_t>(array->length));
2770 25 : aabyBboxStructValidity[i].resize(
2771 25 : static_cast<size_t>(array->length + 7) / 8, 0xFF);
2772 :
2773 25 : bboxStructSchema[i].release = DummyFreeSchema;
2774 25 : bboxStructSchema[i].name = m_apoFieldsBBOX[i]->name().c_str();
2775 25 : bboxStructSchema[i].format = "+s"; // structure
2776 25 : bboxStructSchema[i].flags =
2777 25 : bIsNullable ? ARROW_FLAG_NULLABLE : 0;
2778 25 : bboxStructSchema[i].n_children = BBOX_SUBFIELD_COUNT;
2779 25 : bboxStructSchema[i].children = bboxStructSchemaChildren.data();
2780 :
2781 25 : constexpr int VALIDITY_ARRAY_IDX = 0;
2782 25 : constexpr int BBOX_SUBFIELD_FLOAT_VALUE_IDX = 1;
2783 25 : bboxBuffersPtr[i][XMIN_IDX][BBOX_SUBFIELD_FLOAT_VALUE_IDX] =
2784 25 : aadfMinX[i].data();
2785 25 : bboxBuffersPtr[i][YMIN_IDX][BBOX_SUBFIELD_FLOAT_VALUE_IDX] =
2786 25 : aadfMinY[i].data();
2787 25 : bboxBuffersPtr[i][XMAX_IDX][BBOX_SUBFIELD_FLOAT_VALUE_IDX] =
2788 25 : aadfMaxX[i].data();
2789 25 : bboxBuffersPtr[i][YMAX_IDX][BBOX_SUBFIELD_FLOAT_VALUE_IDX] =
2790 25 : aadfMaxY[i].data();
2791 :
2792 125 : for (int j = 0; j < BBOX_SUBFIELD_COUNT; ++j)
2793 : {
2794 100 : bboxBuffersPtr[i][j][VALIDITY_ARRAY_IDX] = nullptr;
2795 :
2796 100 : bboxArrays[i][j].release = DummyFreeArray;
2797 100 : bboxArrays[i][j].length = array->length;
2798 100 : bboxArrays[i][j].n_buffers = BBOX_SUBFIELD_BUFFER_COUNT;
2799 100 : bboxArrays[i][j].buffers = bboxBuffersPtr[i][j].data();
2800 :
2801 100 : bboxArraysPtr[i][j] = &bboxArrays[i][j];
2802 : }
2803 :
2804 25 : bboxStructArray[i].release = DummyFreeArray;
2805 25 : bboxStructArray[i].n_children = BBOX_SUBFIELD_COUNT;
2806 : // coverity[escape]
2807 25 : bboxStructArray[i].children = bboxArraysPtr[i].data();
2808 25 : bboxStructArray[i].length = array->length;
2809 25 : bboxStructArray[i].n_buffers = BBOX_STRUCT_BUFFER_COUNT;
2810 25 : bboxStructBuffersPtr[i][VALIDITY_ARRAY_IDX] =
2811 25 : bIsNullable ? aabyBboxStructValidity[i].data() : nullptr;
2812 : // coverity[escape]
2813 25 : bboxStructArray[i].buffers = bboxStructBuffersPtr[i].data();
2814 :
2815 25 : newSchemaChildren.emplace_back(&bboxStructSchema[i]);
2816 25 : newArrayChildren.emplace_back(&bboxStructArray[i]);
2817 : }
2818 : }
2819 0 : catch (const std::bad_alloc &)
2820 : {
2821 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
2822 : "Out of memory in "
2823 : "OGRArrowWriterLayer::WriteArrowBatchInternal()");
2824 0 : return false;
2825 : }
2826 : }
2827 :
2828 1760 : for (int i = 0; i < static_cast<int>(schema->n_children); ++i)
2829 : {
2830 1623 : if (!cpl::contains(oSetReferencedFieldsInArraySchema, i))
2831 : {
2832 4 : if (m_osFIDColumn.empty() &&
2833 2 : strcmp(schema->children[i]->name, pszFIDName) == 0)
2834 : {
2835 : // If the input data contains a FID column, but the output data
2836 : // does not, then ignore it.
2837 : }
2838 : else
2839 : {
2840 0 : CPLError(CE_Failure, CPLE_AppDefined,
2841 : "Found field '%s' in array schema that does not exist "
2842 : "in layer schema",
2843 0 : schema->children[i]->name);
2844 0 : return false;
2845 : }
2846 : }
2847 : }
2848 :
2849 : // ImportSchema() would release the schema, but we don't want that
2850 : // So copy the structure content into a local variable, and override its
2851 : // release callback to a no-op. This may be a bit fragile, but it doesn't
2852 : // look like ImportSchema implementation tries to access the C ArrowSchema
2853 : // after it has been called.
2854 137 : struct ArrowSchema lSchema = *schema;
2855 137 : schema = &lSchema;
2856 137 : CPL_IGNORE_RET_VAL(schema);
2857 :
2858 137 : lSchema.n_children = newSchemaChildren.size();
2859 137 : lSchema.children = newSchemaChildren.data();
2860 :
2861 137 : lSchema.release = DummyFreeSchema;
2862 274 : auto poSchemaResult = arrow::ImportSchema(&lSchema);
2863 137 : CPLAssert(lSchema.release == nullptr);
2864 137 : if (!poSchemaResult.ok())
2865 : {
2866 0 : CPLError(CE_Failure, CPLE_AppDefined, "ImportSchema() failed with %s",
2867 0 : poSchemaResult.status().message().c_str());
2868 0 : return false;
2869 : }
2870 274 : auto poSchema = *poSchemaResult;
2871 :
2872 : // Hack the array to use the new children we've computed above
2873 : // but make sure the original release() callback sees the original children
2874 : struct ArrayReleaser
2875 : {
2876 : struct ArrowArray ori_array{};
2877 :
2878 137 : explicit ArrayReleaser(struct ArrowArray *array)
2879 137 : {
2880 137 : memcpy(&ori_array, array, sizeof(*array));
2881 137 : array->release = ArrayReleaser::release;
2882 137 : array->private_data = this;
2883 137 : }
2884 :
2885 137 : static void release(struct ArrowArray *array)
2886 : {
2887 137 : struct ArrayReleaser *releaser =
2888 : static_cast<struct ArrayReleaser *>(array->private_data);
2889 137 : memcpy(array, &(releaser->ori_array), sizeof(*array));
2890 137 : CPLAssert(array->release != nullptr);
2891 137 : array->release(array);
2892 137 : CPLAssert(array->release == nullptr);
2893 137 : delete releaser;
2894 137 : }
2895 : };
2896 :
2897 : // Must be allocated on the heap, since ArrayReleaser::release() will be
2898 : // called after this method has ended.
2899 137 : ArrayReleaser *releaser = new ArrayReleaser(array);
2900 137 : array->private_data = releaser;
2901 137 : array->n_children = newArrayChildren.size();
2902 : // cppcheck-suppress autoVariables
2903 137 : array->children = newArrayChildren.data();
2904 :
2905 : // Process geometry columns:
2906 : // - if the output encoding is WKB, then just note the geometry type and
2907 : // envelope.
2908 : // - otherwise convert to the output encoding.
2909 137 : int nBuilderIdx = 0;
2910 137 : if (!m_osFIDColumn.empty())
2911 : {
2912 2 : nBuilderIdx++;
2913 : }
2914 : std::map<std::string, std::shared_ptr<arrow::Array>>
2915 274 : oMapGeomFieldNameToArray;
2916 274 : for (int i = 0; i < nGeomFieldCount; ++i, ++nBuilderIdx)
2917 : {
2918 : const char *pszThisGeomFieldName =
2919 137 : m_poFeatureDefn->GetGeomFieldDefn(i)->GetNameRef();
2920 137 : int nIdx = poSchema->GetFieldIndex(pszThisGeomFieldName);
2921 137 : if (nIdx < 0)
2922 : {
2923 6 : if (nGeomFieldCount == 1)
2924 6 : nIdx = poSchema->GetFieldIndex(pszSingleGeomFieldName);
2925 6 : if (nIdx < 0)
2926 : {
2927 0 : CPLError(CE_Failure, CPLE_AppDefined,
2928 : "Cannot find geometry field '%s' in schema",
2929 : pszThisGeomFieldName);
2930 0 : return false;
2931 : }
2932 : }
2933 :
2934 137 : if (strcmp(lSchema.children[nIdx]->format, "z") != 0 &&
2935 1 : strcmp(lSchema.children[nIdx]->format, "Z") != 0)
2936 : {
2937 0 : CPLError(CE_Failure, CPLE_AppDefined,
2938 : "Type of geometry field '%s' is not binary, but '%s'",
2939 0 : pszThisGeomFieldName, lSchema.children[nIdx]->format);
2940 0 : return false;
2941 : }
2942 :
2943 137 : const auto psGeomArray = array->children[nIdx];
2944 137 : const uint8_t *pabyValidity =
2945 137 : psGeomArray->null_count != 0
2946 137 : ? static_cast<const uint8_t *>(psGeomArray->buffers[0])
2947 : : nullptr;
2948 137 : const bool bUseOffsets32 =
2949 137 : (strcmp(lSchema.children[nIdx]->format, "z") == 0);
2950 137 : const uint32_t *panOffsets32 =
2951 137 : static_cast<const uint32_t *>(psGeomArray->buffers[1]) +
2952 137 : psGeomArray->offset;
2953 137 : const uint64_t *panOffsets64 =
2954 137 : static_cast<const uint64_t *>(psGeomArray->buffers[1]) +
2955 137 : psGeomArray->offset;
2956 137 : GByte *pabyData =
2957 137 : static_cast<GByte *>(const_cast<void *>(psGeomArray->buffers[2]));
2958 137 : OGREnvelope sEnvelope;
2959 137 : auto poBuilder = m_apoBuilders[nBuilderIdx].get();
2960 :
2961 712 : for (size_t iRow = 0; iRow < static_cast<size_t>(psGeomArray->length);
2962 : ++iRow)
2963 : {
2964 575 : bool bValidGeom = false;
2965 :
2966 1091 : if (!pabyValidity ||
2967 516 : TestBit(pabyValidity,
2968 516 : static_cast<size_t>(iRow + psGeomArray->offset)))
2969 : {
2970 447 : const auto nLen =
2971 447 : bUseOffsets32 ? static_cast<size_t>(panOffsets32[iRow + 1] -
2972 437 : panOffsets32[iRow])
2973 10 : : static_cast<size_t>(panOffsets64[iRow + 1] -
2974 10 : panOffsets64[iRow]);
2975 447 : GByte *pabyWkb =
2976 447 : pabyData + (bUseOffsets32
2977 437 : ? panOffsets32[iRow]
2978 10 : : static_cast<size_t>(panOffsets64[iRow]));
2979 447 : if (m_aeGeomEncoding[i] == OGRArrowGeomEncoding::WKB)
2980 : {
2981 179 : FixupWKBGeometryBeforeWriting(pabyWkb, nLen);
2982 :
2983 179 : uint32_t nType = 0;
2984 179 : bool bNeedSwap = false;
2985 179 : if (OGRWKBGetGeomType(pabyWkb, nLen, bNeedSwap, nType))
2986 : {
2987 179 : m_oSetWrittenGeometryTypes[i].insert(
2988 179 : static_cast<OGRwkbGeometryType>(nType));
2989 179 : if (OGRWKBGetBoundingBox(pabyWkb, nLen, sEnvelope))
2990 : {
2991 179 : bValidGeom = true;
2992 179 : m_aoEnvelopes[i].Merge(sEnvelope);
2993 :
2994 179 : if (m_bWriteBBoxStruct)
2995 : {
2996 51 : aadfMinX[i].push_back(
2997 51 : castToFloatDown(sEnvelope.MinX));
2998 51 : aadfMinY[i].push_back(
2999 51 : castToFloatDown(sEnvelope.MinY));
3000 51 : aadfMaxX[i].push_back(
3001 51 : castToFloatUp(sEnvelope.MaxX));
3002 51 : aadfMaxY[i].push_back(
3003 51 : castToFloatUp(sEnvelope.MaxY));
3004 : }
3005 : }
3006 : }
3007 : }
3008 : else
3009 : {
3010 268 : size_t nBytesConsumedOut = 0;
3011 268 : OGRGeometry *poGeometry = nullptr;
3012 268 : OGRGeometryFactory::createFromWkb(
3013 : pabyWkb, nullptr, &poGeometry, nLen, wkbVariantIso,
3014 : nBytesConsumedOut);
3015 268 : if (BuildGeometry(poGeometry, i, poBuilder) != OGRERR_NONE)
3016 : {
3017 0 : delete poGeometry;
3018 0 : return false;
3019 : }
3020 268 : bValidGeom = true;
3021 268 : if (m_bWriteBBoxStruct)
3022 : {
3023 0 : poGeometry->getEnvelope(&sEnvelope);
3024 0 : aadfMinX[i].push_back(castToFloatDown(sEnvelope.MinX));
3025 0 : aadfMinY[i].push_back(castToFloatDown(sEnvelope.MinY));
3026 0 : aadfMaxX[i].push_back(castToFloatUp(sEnvelope.MaxX));
3027 0 : aadfMaxY[i].push_back(castToFloatUp(sEnvelope.MaxY));
3028 : }
3029 268 : delete poGeometry;
3030 : }
3031 : }
3032 : else
3033 : {
3034 128 : if (m_aeGeomEncoding[i] != OGRArrowGeomEncoding::WKB)
3035 : {
3036 81 : if (BuildGeometry(nullptr, i, poBuilder) != OGRERR_NONE)
3037 0 : return false;
3038 : }
3039 : }
3040 :
3041 575 : if (!bValidGeom && m_bWriteBBoxStruct)
3042 : {
3043 18 : if ((bboxStructSchema[i].flags & ARROW_FLAG_NULLABLE))
3044 : {
3045 18 : bboxStructArray[i].null_count++;
3046 18 : aabyBboxStructValidity[i][iRow / 8] &=
3047 18 : ~(1 << static_cast<int>(iRow % 8));
3048 : }
3049 18 : aadfMinX[i].push_back(0.0f);
3050 18 : aadfMinY[i].push_back(0.0f);
3051 18 : aadfMaxX[i].push_back(0.0f);
3052 18 : aadfMaxY[i].push_back(0.0f);
3053 : }
3054 : }
3055 :
3056 137 : if (m_aeGeomEncoding[i] != OGRArrowGeomEncoding::WKB)
3057 : {
3058 0 : std::shared_ptr<arrow::Array> geomArray;
3059 81 : auto status = poBuilder->Finish(&geomArray);
3060 81 : if (!status.ok())
3061 : {
3062 0 : CPLError(CE_Failure, CPLE_AppDefined,
3063 : "builder::Finish() for field %s failed with %s",
3064 0 : pszThisGeomFieldName, status.message().c_str());
3065 0 : return false;
3066 : }
3067 162 : oMapGeomFieldNameToArray[pszThisGeomFieldName] =
3068 162 : std::move(geomArray);
3069 : }
3070 : }
3071 :
3072 : auto poRecordBatchResult =
3073 274 : arrow::ImportRecordBatch(array, std::move(poSchema));
3074 137 : if (!poRecordBatchResult.ok())
3075 : {
3076 0 : CPLError(CE_Failure, CPLE_AppDefined,
3077 : "ImportRecordBatch() failed with %s",
3078 0 : poRecordBatchResult.status().message().c_str());
3079 0 : return false;
3080 : }
3081 274 : auto poRecordBatch = *poRecordBatchResult;
3082 :
3083 137 : if (!(bRebuildBatch || !oMapGeomFieldNameToArray.empty()))
3084 : {
3085 1106 : for (int i = 0; i < m_poSchema->num_fields(); ++i)
3086 : {
3087 : const auto oIter =
3088 1056 : oMapGeomFieldNameToArray.find(m_poSchema->field(i)->name());
3089 1056 : auto l_array = (oIter != oMapGeomFieldNameToArray.end())
3090 0 : ? oIter->second
3091 2112 : : poRecordBatch->column(i);
3092 2112 : const auto schemaType = m_poSchema->field(i)->type();
3093 2112 : const auto arrayType = l_array->type();
3094 2104 : if (schemaType->id() != arrow::Type::EXTENSION &&
3095 1048 : arrayType->id() == arrow::Type::EXTENSION)
3096 : {
3097 0 : bRebuildBatch = true;
3098 : }
3099 1056 : else if (schemaType->id() != arrayType->id())
3100 : {
3101 3 : CPLDebug(
3102 : "Arrow",
3103 : "Field idx=%d name='%s', schema type=%s, array type=%s", i,
3104 1 : m_poSchema->field(i)->name().c_str(),
3105 2 : schemaType->ToString().c_str(),
3106 2 : arrayType->ToString().c_str());
3107 : }
3108 : }
3109 : }
3110 :
3111 : // below assertion commented out since it is not strictly necessary, but
3112 : // reflects what ImportRecordBatch() does.
3113 : // CPLAssert(array->release == nullptr);
3114 :
3115 : // We may need to reconstruct a final record batch that perfectly matches
3116 : // the expected schema.
3117 137 : if (bRebuildBatch || !oMapGeomFieldNameToArray.empty())
3118 : {
3119 87 : std::vector<std::shared_ptr<arrow::Array>> apoArrays;
3120 677 : for (int i = 0; i < m_poSchema->num_fields(); ++i)
3121 : {
3122 : const auto oIter =
3123 590 : oMapGeomFieldNameToArray.find(m_poSchema->field(i)->name());
3124 590 : if (oIter != oMapGeomFieldNameToArray.end())
3125 81 : apoArrays.emplace_back(oIter->second);
3126 : else
3127 509 : apoArrays.emplace_back(poRecordBatch->column(i));
3128 :
3129 590 : auto expectedFieldType = m_poSchema->field(i)->type();
3130 590 : if (expectedFieldType->id() == arrow::Type::EXTENSION)
3131 : {
3132 0 : auto extensionType = cpl::down_cast<arrow::ExtensionType *>(
3133 : expectedFieldType.get());
3134 0 : expectedFieldType = extensionType->storage_type();
3135 : }
3136 :
3137 590 : if (apoArrays.back()->type()->id() == arrow::Type::EXTENSION)
3138 : {
3139 0 : apoArrays.back() =
3140 0 : std::static_pointer_cast<arrow::ExtensionArray>(
3141 0 : apoArrays.back())
3142 0 : ->storage();
3143 : }
3144 :
3145 590 : if (apoArrays.back()->type()->id() != expectedFieldType->id())
3146 : {
3147 0 : CPLError(
3148 : CE_Failure, CPLE_AppDefined,
3149 : "Field '%s' of unexpected type. Got '%s', expected '%s'",
3150 0 : m_poSchema->field(i)->name().c_str(),
3151 0 : apoArrays.back()->type()->name().c_str(),
3152 0 : expectedFieldType->name().c_str());
3153 0 : return false;
3154 : }
3155 : }
3156 348 : poRecordBatchResult = arrow::RecordBatch::Make(
3157 261 : m_poSchema, poRecordBatch->num_rows(), std::move(apoArrays));
3158 87 : if (!poRecordBatchResult.ok())
3159 : {
3160 0 : CPLError(CE_Failure, CPLE_AppDefined,
3161 : "RecordBatch::Make() failed with %s",
3162 0 : poRecordBatchResult.status().message().c_str());
3163 0 : return false;
3164 : }
3165 87 : poRecordBatch = *poRecordBatchResult;
3166 : }
3167 :
3168 137 : if (writeBatch(poRecordBatch))
3169 : {
3170 137 : m_nFeatureCount += poRecordBatch->num_rows();
3171 137 : return true;
3172 : }
3173 0 : return false;
3174 : #endif
3175 : }
3176 :
3177 : #endif /* OGARROWWRITERLAYER_HPP_INCLUDED */
|
