Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: Arrow generic code
4 : * Purpose: Arrow generic code
5 : * Author: Even Rouault, <even.rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2022, Planet Labs
9 : *
10 : * SPDX-License-Identifier: MIT
11 : ****************************************************************************/
12 :
13 : #ifndef OGARROWWRITERLAYER_HPP_INCLUDED
14 : #define OGARROWWRITERLAYER_HPP_INCLUDED
15 :
16 : #include "ogr_arrow.h"
17 :
18 : #include "cpl_json.h"
19 : #include "cpl_time.h"
20 :
21 : #include "ogrlayerarrow.h"
22 : #include "ogr_wkb.h"
23 :
24 : #include <array>
25 : #include <cinttypes>
26 : #include <limits>
27 :
28 : static constexpr int TZFLAG_UNINITIALIZED = -1;
29 :
30 : #define OGR_ARROW_RETURN_NOT_OK(status, ret_value) \
31 : do \
32 : { \
33 : if (!(status).ok()) \
34 : { \
35 : CPLError(CE_Failure, CPLE_AppDefined, "%s failed", \
36 : ARROW_STRINGIFY(status)); \
37 : return (ret_value); \
38 : } \
39 : } while (false)
40 :
41 : #define OGR_ARROW_RETURN_FALSE_NOT_OK(status) \
42 : OGR_ARROW_RETURN_NOT_OK(status, false)
43 :
44 : #define OGR_ARROW_RETURN_OGRERR_NOT_OK(status) \
45 : OGR_ARROW_RETURN_NOT_OK(status, OGRERR_FAILURE)
46 :
47 : #define OGR_ARROW_PROPAGATE_OGRERR(ret_value) \
48 : do \
49 : { \
50 : if ((ret_value) != OGRERR_NONE) \
51 : return OGRERR_FAILURE; \
52 : } while (0)
53 :
54 : /************************************************************************/
55 : /* OGRArrowWriterLayer() */
56 : /************************************************************************/
57 :
58 499 : inline OGRArrowWriterLayer::OGRArrowWriterLayer(
59 : arrow::MemoryPool *poMemoryPool,
60 : const std::shared_ptr<arrow::io::OutputStream> &poOutputStream,
61 499 : const char *pszLayerName)
62 499 : : m_poMemoryPool(poMemoryPool), m_poOutputStream(poOutputStream)
63 : {
64 499 : m_poFeatureDefn = new OGRFeatureDefn(pszLayerName);
65 499 : m_poFeatureDefn->SetGeomType(wkbNone);
66 499 : m_poFeatureDefn->Reference();
67 499 : SetDescription(pszLayerName);
68 499 : }
69 :
70 : /************************************************************************/
71 : /* ~OGRArrowWriterLayer() */
72 : /************************************************************************/
73 :
74 499 : inline OGRArrowWriterLayer::~OGRArrowWriterLayer()
75 : {
76 499 : CPLDebug("ARROW", "Memory pool (writer layer): bytes_allocated = %" PRId64,
77 499 : m_poMemoryPool->bytes_allocated());
78 499 : CPLDebug("ARROW", "Memory pool (writer layer): max_memory = %" PRId64,
79 499 : m_poMemoryPool->max_memory());
80 :
81 499 : m_poFeatureDefn->Release();
82 499 : }
83 :
84 : /************************************************************************/
85 : /* FinalizeWriting() */
86 : /************************************************************************/
87 :
88 487 : inline bool OGRArrowWriterLayer::FinalizeWriting()
89 : {
90 487 : bool ret = true;
91 :
92 487 : if (!IsFileWriterCreated())
93 : {
94 356 : CreateWriter();
95 : }
96 487 : if (IsFileWriterCreated())
97 : {
98 487 : PerformStepsBeforeFinalFlushGroup();
99 :
100 487 : if (!m_apoBuilders.empty() && m_apoFieldsFromArrowSchema.empty())
101 309 : ret = FlushGroup();
102 :
103 487 : if (!CloseFileWriter())
104 0 : ret = false;
105 : }
106 :
107 487 : return ret;
108 : }
109 :
110 : /************************************************************************/
111 : /* RemoveIDFromMemberOfEnsembles() */
112 : /************************************************************************/
113 :
114 : /* static */
115 : inline void
116 704 : OGRArrowWriterLayer::RemoveIDFromMemberOfEnsembles(CPLJSONObject &obj)
117 : {
118 : // Remove "id" from members of datum ensembles for compatibility with
119 : // older PROJ versions
120 : // Cf https://github.com/opengeospatial/geoparquet/discussions/110
121 : // and https://github.com/OSGeo/PROJ/pull/3221
122 704 : if (obj.GetType() == CPLJSONObject::Type::Object)
123 : {
124 898 : for (auto &subObj : obj.GetChildren())
125 : {
126 684 : RemoveIDFromMemberOfEnsembles(subObj);
127 : }
128 : }
129 542 : else if (obj.GetType() == CPLJSONObject::Type::Array &&
130 542 : obj.GetName() == "members")
131 : {
132 0 : for (auto &subObj : obj.ToArray())
133 : {
134 0 : subObj.Delete("id");
135 : }
136 : }
137 704 : }
138 :
139 : /************************************************************************/
140 : /* IdentifyCRS() */
141 : /************************************************************************/
142 :
143 : /* static */
144 : inline OGRSpatialReference
145 34 : OGRArrowWriterLayer::IdentifyCRS(const OGRSpatialReference *poSRS)
146 : {
147 34 : OGRSpatialReference oSRSIdentified(*poSRS);
148 :
149 34 : if (poSRS->GetAuthorityName(nullptr) == nullptr)
150 : {
151 : // Try to find a registered CRS that matches the input one
152 4 : int nEntries = 0;
153 4 : int *panConfidence = nullptr;
154 : OGRSpatialReferenceH *pahSRS =
155 4 : poSRS->FindMatches(nullptr, &nEntries, &panConfidence);
156 :
157 : // If there are several matches >= 90%, take the only one
158 : // that is EPSG
159 4 : int iOtherAuthority = -1;
160 4 : int iEPSG = -1;
161 4 : const char *const apszOptions[] = {
162 : "IGNORE_DATA_AXIS_TO_SRS_AXIS_MAPPING=YES", nullptr};
163 4 : int iConfidenceBestMatch = -1;
164 6 : for (int iSRS = 0; iSRS < nEntries; iSRS++)
165 : {
166 4 : auto poCandidateCRS = OGRSpatialReference::FromHandle(pahSRS[iSRS]);
167 4 : if (panConfidence[iSRS] < iConfidenceBestMatch ||
168 4 : panConfidence[iSRS] < 70)
169 : {
170 : break;
171 : }
172 3 : if (poSRS->IsSame(poCandidateCRS, apszOptions))
173 : {
174 : const char *pszAuthName =
175 3 : poCandidateCRS->GetAuthorityName(nullptr);
176 3 : if (pszAuthName != nullptr && EQUAL(pszAuthName, "EPSG"))
177 : {
178 2 : iOtherAuthority = -2;
179 2 : if (iEPSG < 0)
180 : {
181 2 : iConfidenceBestMatch = panConfidence[iSRS];
182 2 : iEPSG = iSRS;
183 : }
184 : else
185 : {
186 0 : iEPSG = -1;
187 0 : break;
188 : }
189 : }
190 1 : else if (iEPSG < 0 && pszAuthName != nullptr)
191 : {
192 1 : if (EQUAL(pszAuthName, "OGC"))
193 : {
194 : const char *pszAuthCode =
195 1 : poCandidateCRS->GetAuthorityCode(nullptr);
196 1 : if (pszAuthCode && EQUAL(pszAuthCode, "CRS84"))
197 : {
198 1 : iOtherAuthority = iSRS;
199 1 : break;
200 : }
201 : }
202 0 : else if (iOtherAuthority == -1)
203 : {
204 0 : iConfidenceBestMatch = panConfidence[iSRS];
205 0 : iOtherAuthority = iSRS;
206 : }
207 : else
208 0 : iOtherAuthority = -2;
209 : }
210 : }
211 : }
212 4 : if (iEPSG >= 0)
213 : {
214 2 : oSRSIdentified = *OGRSpatialReference::FromHandle(pahSRS[iEPSG]);
215 : }
216 2 : else if (iOtherAuthority >= 0)
217 : {
218 : oSRSIdentified =
219 1 : *OGRSpatialReference::FromHandle(pahSRS[iOtherAuthority]);
220 : }
221 4 : OSRFreeSRSArray(pahSRS);
222 4 : CPLFree(panConfidence);
223 : }
224 :
225 34 : return oSRSIdentified;
226 : }
227 :
228 : /************************************************************************/
229 : /* CreateSchemaCommon() */
230 : /************************************************************************/
231 :
232 487 : inline void OGRArrowWriterLayer::CreateSchemaCommon()
233 : {
234 487 : CPLAssert(static_cast<int>(m_aeGeomEncoding.size()) ==
235 : m_poFeatureDefn->GetGeomFieldCount());
236 :
237 974 : std::vector<std::shared_ptr<arrow::Field>> fields;
238 487 : bool bNeedGDALSchema = false;
239 :
240 487 : m_anTZFlag.resize(m_poFeatureDefn->GetFieldCount(), TZFLAG_UNINITIALIZED);
241 :
242 487 : if (!m_osFIDColumn.empty())
243 : {
244 21 : bNeedGDALSchema = true;
245 21 : fields.emplace_back(arrow::field(m_osFIDColumn, arrow::int64(), false));
246 : }
247 :
248 487 : if (!m_apoFieldsFromArrowSchema.empty())
249 : {
250 119 : fields.insert(fields.end(), m_apoFieldsFromArrowSchema.begin(),
251 238 : m_apoFieldsFromArrowSchema.end());
252 : }
253 :
254 1243 : for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i)
255 : {
256 756 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
257 756 : std::shared_ptr<arrow::DataType> dt;
258 756 : const auto eDT = poFieldDefn->GetType();
259 756 : const auto eSubDT = poFieldDefn->GetSubType();
260 756 : const auto &osDomainName = poFieldDefn->GetDomainName();
261 756 : const OGRFieldDomain *poFieldDomain = nullptr;
262 756 : const int nWidth = poFieldDefn->GetWidth();
263 756 : if (!osDomainName.empty())
264 : {
265 4 : const auto oIter = m_oMapFieldDomains.find(osDomainName);
266 4 : if (oIter == m_oMapFieldDomains.end())
267 : {
268 0 : CPLError(CE_Warning, CPLE_AppDefined,
269 : "Field %s references domain %s, but the later one "
270 : "has not been created",
271 : poFieldDefn->GetNameRef(), osDomainName.c_str());
272 : }
273 : else
274 : {
275 4 : poFieldDomain = oIter->second.get();
276 : }
277 : }
278 756 : switch (eDT)
279 : {
280 76 : case OFTInteger:
281 76 : if (eSubDT == OFSTBoolean)
282 4 : dt = arrow::boolean();
283 72 : else if (eSubDT == OFSTInt16)
284 4 : dt = arrow::int16();
285 : else
286 68 : dt = arrow::int32();
287 76 : if (poFieldDomain != nullptr)
288 : {
289 4 : dt = arrow::dictionary(dt, arrow::utf8());
290 : }
291 76 : break;
292 :
293 51 : case OFTInteger64:
294 51 : dt = arrow::int64();
295 51 : if (poFieldDomain != nullptr)
296 : {
297 0 : dt = arrow::dictionary(dt, arrow::utf8());
298 : }
299 51 : break;
300 :
301 68 : case OFTReal:
302 : {
303 68 : const int nPrecision = poFieldDefn->GetPrecision();
304 68 : if (nWidth != 0 && nPrecision != 0)
305 : {
306 : // Since arrow 18.0, we could use arrow::smallest_decimal()
307 : // to return the smallest representation (i.e. possibly
308 : // decimal32 and decimal64). But for now keep decimal128
309 : // as the minimum for backwards compatibility.
310 : // GetValueDecimal() and other functions in
311 : // ogrlayerarrow.cpp would have to be adapted for decimal32
312 : // and decimal64 compatibility.
313 11 : if (nWidth > 38)
314 0 : dt = arrow::decimal256(nWidth, nPrecision);
315 : else
316 11 : dt = arrow::decimal128(nWidth, nPrecision);
317 : }
318 57 : else if (eSubDT == OFSTFloat32)
319 7 : dt = arrow::float32();
320 : else
321 50 : dt = arrow::float64();
322 68 : break;
323 : }
324 :
325 341 : case OFTString:
326 : case OFTWideString:
327 341 : if ((eSubDT != OFSTNone && eSubDT != OFSTJSON) || nWidth > 0)
328 3 : bNeedGDALSchema = true;
329 341 : dt = arrow::utf8();
330 341 : break;
331 :
332 19 : case OFTBinary:
333 19 : if (nWidth != 0)
334 4 : dt = arrow::fixed_size_binary(nWidth);
335 : else
336 15 : dt = arrow::binary();
337 19 : break;
338 :
339 48 : case OFTIntegerList:
340 48 : if (eSubDT == OFSTBoolean)
341 8 : dt = arrow::list(arrow::boolean());
342 40 : else if (eSubDT == OFSTInt16)
343 0 : dt = arrow::list(arrow::int16());
344 : else
345 40 : dt = arrow::list(arrow::int32());
346 48 : break;
347 :
348 20 : case OFTInteger64List:
349 20 : dt = arrow::list(arrow::int64());
350 20 : break;
351 :
352 35 : case OFTRealList:
353 35 : if (eSubDT == OFSTFloat32)
354 11 : dt = arrow::list(arrow::float32());
355 : else
356 24 : dt = arrow::list(arrow::float64());
357 35 : break;
358 :
359 12 : case OFTStringList:
360 : case OFTWideStringList:
361 12 : dt = arrow::list(arrow::utf8());
362 12 : break;
363 :
364 31 : case OFTDate:
365 31 : dt = arrow::date32();
366 31 : break;
367 :
368 8 : case OFTTime:
369 8 : dt = arrow::time32(arrow::TimeUnit::MILLI);
370 8 : break;
371 :
372 47 : case OFTDateTime:
373 : {
374 47 : const int nTZFlag = poFieldDefn->GetTZFlag();
375 47 : if (nTZFlag >= OGR_TZFLAG_MIXED_TZ)
376 : {
377 12 : m_anTZFlag[i] = nTZFlag;
378 : }
379 47 : dt = arrow::timestamp(arrow::TimeUnit::MILLI);
380 47 : break;
381 : }
382 : }
383 :
384 756 : auto field = arrow::field(poFieldDefn->GetNameRef(), std::move(dt),
385 2268 : poFieldDefn->IsNullable());
386 756 : if (eDT == OFTString && eSubDT == OFSTJSON)
387 : {
388 83 : auto kvMetadata = std::make_shared<arrow::KeyValueMetadata>();
389 83 : kvMetadata->Append(ARROW_EXTENSION_NAME_KEY,
390 : EXTENSION_NAME_ARROW_JSON);
391 83 : field = field->WithMetadata(kvMetadata);
392 : }
393 :
394 756 : fields.emplace_back(std::move(field));
395 756 : if (poFieldDefn->GetAlternativeNameRef()[0])
396 2 : bNeedGDALSchema = true;
397 756 : if (!poFieldDefn->GetComment().empty())
398 3 : bNeedGDALSchema = true;
399 : }
400 :
401 965 : for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
402 : {
403 478 : const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
404 478 : const auto eGType = poGeomFieldDefn->GetType();
405 : const int nDim =
406 478 : 2 + (OGR_GT_HasZ(eGType) ? 1 : 0) + (OGR_GT_HasM(eGType) ? 1 : 0);
407 :
408 478 : const bool pointFieldNullable = GetDriverUCName() == "PARQUET";
409 :
410 : // Fixed Size List GeoArrow encoding
411 : const auto getFixedSizeListOfPoint =
412 228 : [nDim, eGType, pointFieldNullable]()
413 : {
414 : return arrow::fixed_size_list(
415 146 : arrow::field(nDim == 2 ? "xy"
416 30 : : nDim == 3 ? (OGR_GT_HasZ(eGType) ? "xyz" : "xym")
417 : : "xyzm",
418 : arrow::float64(), pointFieldNullable),
419 116 : nDim);
420 478 : };
421 :
422 : // Struct GeoArrow encoding
423 1434 : auto xField(arrow::field("x", arrow::float64(), false));
424 1434 : auto yField(arrow::field("y", arrow::float64(), false));
425 : std::vector<std::shared_ptr<arrow::Field>> pointFields{
426 : arrow::field("x", arrow::float64(), false),
427 2868 : arrow::field("y", arrow::float64(), false)};
428 478 : if (OGR_GT_HasZ(eGType))
429 : pointFields.emplace_back(
430 145 : arrow::field("z", arrow::float64(), false));
431 478 : if (OGR_GT_HasM(eGType))
432 : pointFields.emplace_back(
433 52 : arrow::field("m", arrow::float64(), false));
434 956 : auto pointStructType(arrow::struct_(std::move(pointFields)));
435 :
436 40 : const auto getListOfVertices = [&getFixedSizeListOfPoint]()
437 : {
438 80 : return arrow::list(std::make_shared<arrow::Field>(
439 120 : "vertices", getFixedSizeListOfPoint()));
440 478 : };
441 :
442 22 : const auto getListOfRings = [&getListOfVertices]()
443 : {
444 : return arrow::list(
445 44 : std::make_shared<arrow::Field>("rings", getListOfVertices()));
446 478 : };
447 :
448 144 : const auto getListOfVerticesStruct = [&pointStructType]()
449 : {
450 : return arrow::list(
451 288 : std::make_shared<arrow::Field>("vertices", pointStructType));
452 478 : };
453 :
454 84 : const auto getListOfRingsStruct = [&getListOfVerticesStruct]()
455 : {
456 168 : return arrow::list(std::make_shared<arrow::Field>(
457 252 : "rings", getListOfVerticesStruct()));
458 478 : };
459 :
460 478 : std::shared_ptr<arrow::DataType> dt;
461 478 : switch (m_aeGeomEncoding[i])
462 : {
463 154 : case OGRArrowGeomEncoding::WKB:
464 : #if ARROW_VERSION_MAJOR >= 21
465 : if (m_bUseArrowWKBExtension)
466 : {
467 : CPLJSONDocument oMetadataDoc;
468 :
469 : const auto poSRS = poGeomFieldDefn->GetSpatialRef();
470 : if (poSRS)
471 : {
472 : OGRSpatialReference oSRSIdentified(IdentifyCRS(poSRS));
473 :
474 : // CRS encoded as PROJJSON
475 : char *pszPROJJSON = nullptr;
476 : oSRSIdentified.exportToPROJJSON(&pszPROJJSON, nullptr);
477 : CPLJSONDocument oCRSDoc;
478 : CPL_IGNORE_RET_VAL(oCRSDoc.LoadMemory(pszPROJJSON));
479 : CPLFree(pszPROJJSON);
480 : CPLJSONObject oCRSRoot = oCRSDoc.GetRoot();
481 : RemoveIDFromMemberOfEnsembles(oCRSRoot);
482 :
483 : oMetadataDoc.GetRoot().Add("crs", oCRSRoot);
484 : }
485 :
486 : if (m_bEdgesSpherical)
487 : {
488 : oMetadataDoc.GetRoot().Add("edges", "spherical");
489 : }
490 :
491 : const std::string metadata = oMetadataDoc.GetRoot().Format(
492 : CPLJSONObject::PrettyFormat::Plain);
493 : dt = std::make_shared<OGRGeoArrowWkbExtensionType>(
494 : arrow::binary(), metadata);
495 : }
496 : else
497 : #endif
498 : {
499 154 : dt = arrow::binary();
500 : }
501 154 : break;
502 :
503 53 : case OGRArrowGeomEncoding::WKT:
504 53 : dt = arrow::utf8();
505 53 : break;
506 :
507 0 : case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
508 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
509 0 : CPLAssert(false);
510 : break;
511 :
512 9 : case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
513 9 : dt = getFixedSizeListOfPoint();
514 9 : break;
515 :
516 9 : case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
517 9 : dt = getListOfVertices();
518 9 : break;
519 :
520 11 : case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
521 11 : dt = getListOfRings();
522 11 : break;
523 :
524 9 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
525 18 : dt = arrow::list(std::make_shared<arrow::Field>(
526 27 : "points", getFixedSizeListOfPoint()));
527 9 : break;
528 :
529 9 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
530 18 : dt = arrow::list(std::make_shared<arrow::Field>(
531 27 : "linestrings", getListOfVertices()));
532 9 : break;
533 :
534 11 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
535 22 : dt = arrow::list(std::make_shared<arrow::Field>(
536 33 : "polygons", getListOfRings()));
537 11 : break;
538 :
539 39 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
540 39 : dt = pointStructType;
541 39 : break;
542 :
543 30 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
544 30 : dt = getListOfVerticesStruct();
545 30 : break;
546 :
547 42 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
548 42 : dt = getListOfRingsStruct();
549 42 : break;
550 :
551 30 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
552 60 : dt = arrow::list(
553 90 : std::make_shared<arrow::Field>("points", pointStructType));
554 30 : break;
555 :
556 30 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
557 60 : dt = arrow::list(std::make_shared<arrow::Field>(
558 90 : "linestrings", getListOfVerticesStruct()));
559 30 : break;
560 :
561 42 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
562 84 : dt = arrow::list(std::make_shared<arrow::Field>(
563 126 : "polygons", getListOfRingsStruct()));
564 42 : break;
565 : }
566 :
567 : std::shared_ptr<arrow::Field> field(
568 478 : arrow::field(poGeomFieldDefn->GetNameRef(), std::move(dt),
569 1434 : poGeomFieldDefn->IsNullable()));
570 478 : if (m_bWriteFieldArrowExtensionName)
571 : {
572 136 : auto kvMetadata = field->metadata()
573 136 : ? field->metadata()->Copy()
574 136 : : std::make_shared<arrow::KeyValueMetadata>();
575 272 : kvMetadata->Append(
576 : ARROW_EXTENSION_NAME_KEY,
577 136 : GetGeomEncodingAsString(m_aeGeomEncoding[i], false));
578 136 : field = field->WithMetadata(kvMetadata);
579 : }
580 :
581 478 : m_apoBaseStructGeomType.emplace_back(std::move(pointStructType));
582 :
583 478 : fields.emplace_back(std::move(field));
584 : }
585 :
586 487 : if (m_bWriteBBoxStruct)
587 : {
588 549 : for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
589 : {
590 273 : const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
591 819 : auto bbox_field_xmin(arrow::field("xmin", arrow::float32(), false));
592 819 : auto bbox_field_ymin(arrow::field("ymin", arrow::float32(), false));
593 819 : auto bbox_field_xmax(arrow::field("xmax", arrow::float32(), false));
594 819 : auto bbox_field_ymax(arrow::field("ymax", arrow::float32(), false));
595 : auto bbox_field(arrow::field(
596 273 : m_oBBoxStructFieldName.empty()
597 924 : ? std::string(poGeomFieldDefn->GetNameRef()).append("_bbox")
598 : : m_oBBoxStructFieldName,
599 1638 : arrow::struct_(
600 273 : {std::move(bbox_field_xmin), std::move(bbox_field_ymin),
601 1638 : std::move(bbox_field_xmax), std::move(bbox_field_ymax)}),
602 1526 : poGeomFieldDefn->IsNullable()));
603 273 : fields.emplace_back(bbox_field);
604 273 : m_apoFieldsBBOX.emplace_back(bbox_field);
605 : }
606 : }
607 :
608 487 : m_aoEnvelopes.resize(m_poFeatureDefn->GetGeomFieldCount());
609 487 : m_oSetWrittenGeometryTypes.resize(m_poFeatureDefn->GetGeomFieldCount());
610 :
611 487 : m_poSchema = arrow::schema(std::move(fields));
612 487 : CPLAssert(m_poSchema);
613 514 : if (bNeedGDALSchema &&
614 27 : CPLTestBool(CPLGetConfigOption(
615 514 : ("OGR_" + GetDriverUCName() + "_WRITE_GDAL_SCHEMA").c_str(),
616 : "YES")))
617 : {
618 54 : CPLJSONObject oRoot;
619 54 : CPLJSONObject oColumns;
620 :
621 27 : if (!m_osFIDColumn.empty())
622 21 : oRoot.Add("fid", m_osFIDColumn);
623 :
624 27 : oRoot.Add("columns", oColumns);
625 221 : for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i)
626 : {
627 194 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
628 388 : CPLJSONObject oColumn;
629 194 : oColumns.Add(poFieldDefn->GetNameRef(), oColumn);
630 194 : oColumn.Add("type", OGR_GetFieldTypeName(poFieldDefn->GetType()));
631 194 : const auto eSubDT = poFieldDefn->GetSubType();
632 194 : if (eSubDT != OFSTNone)
633 56 : oColumn.Add("subtype", OGR_GetFieldSubTypeName(eSubDT));
634 194 : const int nWidth = poFieldDefn->GetWidth();
635 194 : if (nWidth > 0)
636 15 : oColumn.Add("width", nWidth);
637 194 : const int nPrecision = poFieldDefn->GetPrecision();
638 194 : if (nPrecision > 0)
639 7 : oColumn.Add("precision", nPrecision);
640 194 : if (poFieldDefn->GetAlternativeNameRef()[0])
641 2 : oColumn.Add("alternative_name",
642 : poFieldDefn->GetAlternativeNameRef());
643 194 : if (!poFieldDefn->GetComment().empty())
644 3 : oColumn.Add("comment", poFieldDefn->GetComment());
645 : }
646 :
647 27 : auto kvMetadata = m_poSchema->metadata()
648 0 : ? m_poSchema->metadata()->Copy()
649 54 : : std::make_shared<arrow::KeyValueMetadata>();
650 54 : kvMetadata->Append("gdal:schema",
651 54 : oRoot.Format(CPLJSONObject::PrettyFormat::Plain));
652 27 : m_poSchema = m_poSchema->WithMetadata(kvMetadata);
653 27 : CPLAssert(m_poSchema);
654 : }
655 487 : }
656 :
657 : /************************************************************************/
658 : /* FinalizeSchema() */
659 : /************************************************************************/
660 :
661 431 : inline void OGRArrowWriterLayer::FinalizeSchema()
662 : {
663 : // Final tuning of schema taking into actual timezone values
664 : // from features
665 431 : int nArrowIdxFirstField = !m_osFIDColumn.empty() ? 1 : 0;
666 1185 : for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i)
667 : {
668 754 : if (m_anTZFlag[i] >= OGR_TZFLAG_MIXED_TZ)
669 : {
670 12 : const int nOffset = m_anTZFlag[i] == OGR_TZFLAG_UTC
671 12 : ? 0
672 8 : : (m_anTZFlag[i] - OGR_TZFLAG_UTC) * 15;
673 12 : int nHours = static_cast<int>(nOffset / 60); // Round towards zero.
674 12 : const int nMinutes = std::abs(nOffset - nHours * 60);
675 :
676 : const std::string osTZ =
677 : CPLSPrintf("%c%02d:%02d", nOffset >= 0 ? '+' : '-',
678 24 : std::abs(nHours), nMinutes);
679 24 : auto dt = arrow::timestamp(arrow::TimeUnit::MILLI, osTZ);
680 12 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
681 12 : auto field = arrow::field(poFieldDefn->GetNameRef(), std::move(dt),
682 36 : poFieldDefn->IsNullable());
683 24 : auto result = m_poSchema->SetField(nArrowIdxFirstField + i, field);
684 12 : if (!result.ok())
685 : {
686 0 : CPLError(CE_Warning, CPLE_AppDefined,
687 : "Schema::SetField() failed with %s",
688 0 : result.status().message().c_str());
689 : }
690 : else
691 : {
692 12 : m_poSchema = *result;
693 : }
694 : }
695 : }
696 431 : }
697 :
698 : /************************************************************************/
699 : /* AddFieldDomain() */
700 : /************************************************************************/
701 :
702 : inline bool
703 11 : OGRArrowWriterLayer::AddFieldDomain(std::unique_ptr<OGRFieldDomain> &&domain,
704 : std::string &failureReason)
705 : {
706 11 : if (domain->GetDomainType() != OFDT_CODED)
707 : {
708 0 : failureReason = "Only coded field domains are supported by Arrow";
709 0 : return false;
710 : }
711 :
712 : const OGRCodedFieldDomain *poDomain =
713 11 : static_cast<const OGRCodedFieldDomain *>(domain.get());
714 11 : const OGRCodedValue *psIter = poDomain->GetEnumeration();
715 :
716 : auto poStringBuilder =
717 22 : std::make_shared<arrow::StringBuilder>(m_poMemoryPool);
718 :
719 11 : int nLastCode = -1;
720 44 : for (; psIter->pszCode; ++psIter)
721 : {
722 33 : if (CPLGetValueType(psIter->pszCode) != CPL_VALUE_INTEGER)
723 : {
724 0 : failureReason = "Non integer code in domain ";
725 0 : failureReason += domain->GetName();
726 0 : return false;
727 : }
728 33 : int nCode = atoi(psIter->pszCode);
729 33 : if (nCode <= nLastCode || nCode - nLastCode > 100)
730 : {
731 0 : failureReason = "Too sparse codes in domain ";
732 0 : failureReason += domain->GetName();
733 0 : return false;
734 : }
735 33 : for (int i = nLastCode + 1; i < nCode; ++i)
736 : {
737 0 : OGR_ARROW_RETURN_FALSE_NOT_OK(poStringBuilder->AppendNull());
738 : }
739 33 : if (psIter->pszValue)
740 33 : OGR_ARROW_RETURN_FALSE_NOT_OK(
741 : poStringBuilder->Append(psIter->pszValue));
742 : else
743 0 : OGR_ARROW_RETURN_FALSE_NOT_OK(poStringBuilder->AppendNull());
744 33 : nLastCode = nCode;
745 : }
746 :
747 11 : std::shared_ptr<arrow::Array> stringArray;
748 22 : auto status = poStringBuilder->Finish(&stringArray);
749 11 : if (!status.ok())
750 : {
751 0 : CPLError(CE_Failure, CPLE_AppDefined,
752 : "StringArray::Finish() failed with %s",
753 0 : status.message().c_str());
754 0 : return false;
755 : }
756 :
757 11 : m_oMapFieldDomainToStringArray[domain->GetName()] = std::move(stringArray);
758 11 : m_oMapFieldDomains[domain->GetName()] = std::move(domain);
759 11 : return true;
760 : }
761 :
762 : /************************************************************************/
763 : /* GetFieldDomainNames() */
764 : /************************************************************************/
765 :
766 0 : inline std::vector<std::string> OGRArrowWriterLayer::GetFieldDomainNames() const
767 : {
768 0 : std::vector<std::string> names;
769 0 : names.reserve(m_oMapFieldDomains.size());
770 0 : for (const auto &it : m_oMapFieldDomains)
771 : {
772 0 : names.emplace_back(it.first);
773 : }
774 0 : return names;
775 : }
776 :
777 : /************************************************************************/
778 : /* GetFieldDomain() */
779 : /************************************************************************/
780 :
781 : inline const OGRFieldDomain *
782 15 : OGRArrowWriterLayer::GetFieldDomain(const std::string &name) const
783 : {
784 15 : const auto iter = m_oMapFieldDomains.find(name);
785 15 : if (iter == m_oMapFieldDomains.end())
786 11 : return nullptr;
787 4 : return iter->second.get();
788 : }
789 :
790 : /************************************************************************/
791 : /* CreateField() */
792 : /************************************************************************/
793 :
794 757 : inline OGRErr OGRArrowWriterLayer::CreateField(const OGRFieldDefn *poField,
795 : int /* bApproxOK */)
796 : {
797 757 : if (m_poSchema)
798 : {
799 1 : CPLError(CE_Failure, CPLE_NotSupported,
800 : "Cannot add field after a first feature has been written");
801 1 : return OGRERR_FAILURE;
802 : }
803 756 : if (!m_apoFieldsFromArrowSchema.empty())
804 : {
805 0 : CPLError(CE_Failure, CPLE_NotSupported,
806 : "Cannot mix calls to CreateField() and "
807 : "CreateFieldFromArrowSchema()");
808 0 : return OGRERR_FAILURE;
809 : }
810 756 : m_poFeatureDefn->AddFieldDefn(poField);
811 756 : return OGRERR_NONE;
812 : }
813 :
814 : /************************************************************************/
815 : /* OGRLayer::CreateFieldFromArrowSchema() */
816 : /************************************************************************/
817 :
818 998 : inline bool OGRArrowWriterLayer::CreateFieldFromArrowSchema(
819 : const struct ArrowSchema *schema, CSLConstList /*papszOptions*/)
820 : {
821 998 : if (m_poSchema)
822 : {
823 0 : CPLError(CE_Failure, CPLE_NotSupported,
824 : "Cannot add field after a first feature has been written");
825 0 : return false;
826 : }
827 :
828 998 : if (m_poFeatureDefn->GetFieldCount())
829 : {
830 0 : CPLError(CE_Failure, CPLE_NotSupported,
831 : "Cannot mix calls to CreateField() and "
832 : "CreateFieldFromArrowSchema()");
833 0 : return false;
834 : }
835 :
836 998 : if (m_osFIDColumn == schema->name)
837 : {
838 0 : CPLError(CE_Failure, CPLE_AppDefined,
839 : "FID column has the same name as this field: %s",
840 0 : schema->name);
841 0 : return false;
842 : }
843 :
844 35951 : for (auto &apoField : m_apoFieldsFromArrowSchema)
845 : {
846 34953 : if (apoField->name() == schema->name)
847 : {
848 0 : CPLError(CE_Failure, CPLE_AppDefined,
849 0 : "Field of name %s already exists", schema->name);
850 0 : return false;
851 : }
852 : }
853 :
854 998 : if (m_poFeatureDefn->GetGeomFieldIndex(schema->name) >= 0)
855 : {
856 0 : CPLError(CE_Failure, CPLE_AppDefined,
857 0 : "Geometry field of name %s already exists", schema->name);
858 0 : return false;
859 : }
860 :
861 : // ImportField() would release the schema, but we don't want that
862 : // So copy the structure content into a local variable, and override its
863 : // release callback to a no-op. This may be a bit fragile, but it doesn't
864 : // look like ImportField implementation tries to access the C ArrowSchema
865 : // after it has been called.
866 998 : struct ArrowSchema lSchema = *schema;
867 998 : const auto DummyFreeSchema = [](struct ArrowSchema *ptrSchema)
868 998 : { ptrSchema->release = nullptr; };
869 998 : lSchema.release = DummyFreeSchema;
870 1996 : auto result = arrow::ImportField(&lSchema);
871 998 : CPLAssert(lSchema.release == nullptr);
872 998 : if (!result.ok())
873 : {
874 0 : CPLError(CE_Failure, CPLE_AppDefined,
875 : "CreateFieldFromArrowSchema() failed");
876 0 : return false;
877 : }
878 998 : m_apoFieldsFromArrowSchema.emplace_back(std::move(*result));
879 998 : return true;
880 : }
881 :
882 : /************************************************************************/
883 : /* GetPreciseArrowGeomEncoding() */
884 : /************************************************************************/
885 :
886 273 : inline OGRArrowGeomEncoding OGRArrowWriterLayer::GetPreciseArrowGeomEncoding(
887 : OGRArrowGeomEncoding eEncodingType, OGRwkbGeometryType eGType)
888 : {
889 273 : CPLAssert(eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC ||
890 : eEncodingType == OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC);
891 273 : const auto eFlatType = wkbFlatten(eGType);
892 273 : if (eFlatType == wkbPoint)
893 : {
894 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
895 48 : ? OGRArrowGeomEncoding::GEOARROW_FSL_POINT
896 48 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT;
897 : }
898 225 : else if (eFlatType == wkbLineString)
899 : {
900 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
901 39 : ? OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING
902 39 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING;
903 : }
904 186 : else if (eFlatType == wkbPolygon)
905 : {
906 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
907 53 : ? OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON
908 53 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON;
909 : }
910 133 : else if (eFlatType == wkbMultiPoint)
911 : {
912 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
913 39 : ? OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT
914 39 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT;
915 : }
916 94 : else if (eFlatType == wkbMultiLineString)
917 : {
918 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
919 39 : ? OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING
920 39 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING;
921 : }
922 55 : else if (eFlatType == wkbMultiPolygon)
923 : {
924 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
925 53 : ? OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON
926 53 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON;
927 : }
928 : else
929 : {
930 2 : CPLError(CE_Failure, CPLE_NotSupported,
931 : "GeoArrow encoding is currently not supported for %s",
932 : OGRGeometryTypeToName(eGType));
933 2 : return eEncodingType;
934 : }
935 : }
936 :
937 : /************************************************************************/
938 : /* GetGeomEncodingAsString() */
939 : /************************************************************************/
940 :
941 : inline const char *
942 726 : OGRArrowWriterLayer::GetGeomEncodingAsString(OGRArrowGeomEncoding eGeomEncoding,
943 : bool bForParquetGeo)
944 : {
945 726 : switch (eGeomEncoding)
946 : {
947 201 : case OGRArrowGeomEncoding::WKB:
948 201 : return bForParquetGeo ? "WKB" : "geoarrow.wkb";
949 111 : case OGRArrowGeomEncoding::WKT:
950 111 : return bForParquetGeo ? "WKT" : "geoarrow.wkt";
951 0 : case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
952 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
953 0 : CPLAssert(false);
954 : break;
955 19 : case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
956 19 : return "geoarrow.point";
957 19 : case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
958 19 : return "geoarrow.linestring";
959 21 : case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
960 21 : return "geoarrow.polygon";
961 19 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
962 19 : return "geoarrow.multipoint";
963 19 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
964 19 : return "geoarrow.multilinestring";
965 21 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
966 21 : return "geoarrow.multipolygon";
967 62 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
968 62 : return bForParquetGeo ? "point" : "geoarrow.point";
969 42 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
970 42 : return bForParquetGeo ? "linestring" : "geoarrow.linestring";
971 54 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
972 54 : return bForParquetGeo ? "polygon" : "geoarrow.polygon";
973 42 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
974 42 : return bForParquetGeo ? "multipoint" : "geoarrow.multipoint";
975 42 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
976 42 : return bForParquetGeo ? "multilinestring"
977 42 : : "geoarrow.multilinestring";
978 54 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
979 54 : return bForParquetGeo ? "multipolygon" : "geoarrow.multipolygon";
980 : }
981 0 : return nullptr;
982 : }
983 :
984 : /************************************************************************/
985 : /* CreateGeomField() */
986 : /************************************************************************/
987 :
988 : inline OGRErr
989 27 : OGRArrowWriterLayer::CreateGeomField(const OGRGeomFieldDefn *poField,
990 : int /* bApproxOK */)
991 : {
992 27 : if (m_poSchema)
993 : {
994 1 : CPLError(CE_Failure, CPLE_NotSupported,
995 : "Cannot add field after a first feature has been written");
996 1 : return OGRERR_FAILURE;
997 : }
998 26 : const auto eGType = poField->GetType();
999 26 : if (!IsSupportedGeometryType(eGType))
1000 : {
1001 0 : return OGRERR_FAILURE;
1002 : }
1003 :
1004 26 : if (IsSRSRequired() && poField->GetSpatialRef() == nullptr)
1005 : {
1006 0 : CPLError(CE_Warning, CPLE_AppDefined,
1007 : "Geometry column should have an associated CRS");
1008 : }
1009 26 : auto eGeomEncoding = m_eGeomEncoding;
1010 26 : if (eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC ||
1011 26 : eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC)
1012 : {
1013 0 : const auto eEncodingType = eGeomEncoding;
1014 0 : eGeomEncoding = GetPreciseArrowGeomEncoding(eEncodingType, eGType);
1015 0 : if (eGeomEncoding == eEncodingType)
1016 0 : return OGRERR_FAILURE;
1017 : }
1018 26 : m_aeGeomEncoding.push_back(eGeomEncoding);
1019 26 : m_poFeatureDefn->AddGeomFieldDefn(poField);
1020 26 : return OGRERR_NONE;
1021 : }
1022 :
1023 : /************************************************************************/
1024 : /* MakeGeoArrowBuilder() */
1025 : /************************************************************************/
1026 :
1027 : static std::shared_ptr<arrow::ArrayBuilder>
1028 134 : MakeGeoArrowBuilder(arrow::MemoryPool *poMemoryPool, int nDim, int nDepth)
1029 : {
1030 134 : if (nDepth == 0)
1031 104 : return std::make_shared<arrow::FixedSizeListBuilder>(
1032 104 : poMemoryPool, std::make_shared<arrow::DoubleBuilder>(poMemoryPool),
1033 52 : nDim);
1034 : else
1035 164 : return std::make_shared<arrow::ListBuilder>(
1036 246 : poMemoryPool, MakeGeoArrowBuilder(poMemoryPool, nDim, nDepth - 1));
1037 : }
1038 :
1039 : /************************************************************************/
1040 : /* MakeGeoArrowStructBuilder() */
1041 : /************************************************************************/
1042 :
1043 : static std::shared_ptr<arrow::ArrayBuilder>
1044 532 : MakeGeoArrowStructBuilder(arrow::MemoryPool *poMemoryPool, int nDim, int nDepth,
1045 : const std::shared_ptr<arrow::DataType> &eBaseType)
1046 : {
1047 532 : if (nDepth == 0)
1048 : {
1049 211 : std::vector<std::shared_ptr<arrow::ArrayBuilder>> builders;
1050 729 : for (int i = 0; i < nDim; ++i)
1051 : builders.emplace_back(
1052 518 : std::make_shared<arrow::DoubleBuilder>(poMemoryPool));
1053 422 : return std::make_shared<arrow::StructBuilder>(eBaseType, poMemoryPool,
1054 422 : std::move(builders));
1055 : }
1056 : else
1057 642 : return std::make_shared<arrow::ListBuilder>(
1058 642 : poMemoryPool, MakeGeoArrowStructBuilder(poMemoryPool, nDim,
1059 321 : nDepth - 1, eBaseType));
1060 : }
1061 :
1062 : /************************************************************************/
1063 : /* ClearArrayBuilers() */
1064 : /************************************************************************/
1065 :
1066 350 : inline void OGRArrowWriterLayer::ClearArrayBuilers()
1067 : {
1068 350 : m_apoBuilders.clear();
1069 350 : m_apoBuildersBBOXStruct.clear();
1070 350 : m_apoBuildersBBOXXMin.clear();
1071 350 : m_apoBuildersBBOXYMin.clear();
1072 350 : m_apoBuildersBBOXXMax.clear();
1073 350 : m_apoBuildersBBOXYMax.clear();
1074 350 : }
1075 :
1076 : /************************************************************************/
1077 : /* CreateArrayBuilders() */
1078 : /************************************************************************/
1079 :
1080 469 : inline void OGRArrowWriterLayer::CreateArrayBuilders()
1081 : {
1082 469 : m_apoBuilders.reserve(1 + m_poFeatureDefn->GetFieldCount() +
1083 469 : m_poFeatureDefn->GetGeomFieldCount());
1084 :
1085 469 : int nArrowIdx = 0;
1086 469 : if (!m_osFIDColumn.empty())
1087 : {
1088 51 : m_apoBuilders.emplace_back(std::make_shared<arrow::Int64Builder>());
1089 51 : nArrowIdx++;
1090 : }
1091 :
1092 1941 : for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i, ++nArrowIdx)
1093 : {
1094 1472 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
1095 1472 : std::shared_ptr<arrow::ArrayBuilder> builder;
1096 1472 : const auto eSubDT = poFieldDefn->GetSubType();
1097 1472 : switch (poFieldDefn->GetType())
1098 : {
1099 157 : case OFTInteger:
1100 157 : if (eSubDT == OFSTBoolean)
1101 : builder =
1102 12 : std::make_shared<arrow::BooleanBuilder>(m_poMemoryPool);
1103 145 : else if (eSubDT == OFSTInt16)
1104 : builder =
1105 12 : std::make_shared<arrow::Int16Builder>(m_poMemoryPool);
1106 : else
1107 : builder =
1108 133 : std::make_shared<arrow::Int32Builder>(m_poMemoryPool);
1109 157 : break;
1110 :
1111 99 : case OFTInteger64:
1112 99 : builder = std::make_shared<arrow::Int64Builder>(m_poMemoryPool);
1113 99 : break;
1114 :
1115 122 : case OFTReal:
1116 : {
1117 244 : const auto arrowType = m_poSchema->fields()[nArrowIdx]->type();
1118 122 : if (arrowType->id() == arrow::Type::DECIMAL128)
1119 27 : builder = std::make_shared<arrow::Decimal128Builder>(
1120 27 : arrowType, m_poMemoryPool);
1121 95 : else if (arrowType->id() == arrow::Type::DECIMAL256)
1122 0 : builder = std::make_shared<arrow::Decimal256Builder>(
1123 0 : arrowType, m_poMemoryPool);
1124 95 : else if (eSubDT == OFSTFloat32)
1125 : builder =
1126 21 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool);
1127 : else
1128 : builder =
1129 74 : std::make_shared<arrow::DoubleBuilder>(m_poMemoryPool);
1130 122 : break;
1131 : }
1132 :
1133 540 : case OFTString:
1134 : case OFTWideString:
1135 : builder =
1136 540 : std::make_shared<arrow::StringBuilder>(m_poMemoryPool);
1137 540 : break;
1138 :
1139 43 : case OFTBinary:
1140 43 : if (poFieldDefn->GetWidth() != 0)
1141 24 : builder = std::make_shared<arrow::FixedSizeBinaryBuilder>(
1142 24 : arrow::fixed_size_binary(poFieldDefn->GetWidth()),
1143 24 : m_poMemoryPool);
1144 : else
1145 : builder =
1146 31 : std::make_shared<arrow::BinaryBuilder>(m_poMemoryPool);
1147 43 : break;
1148 :
1149 144 : case OFTIntegerList:
1150 : {
1151 144 : std::shared_ptr<arrow::ArrayBuilder> poBaseBuilder;
1152 144 : if (eSubDT == OFSTBoolean)
1153 : poBaseBuilder =
1154 24 : std::make_shared<arrow::BooleanBuilder>(m_poMemoryPool);
1155 120 : else if (eSubDT == OFSTInt16)
1156 : poBaseBuilder =
1157 0 : std::make_shared<arrow::Int16Builder>(m_poMemoryPool);
1158 : else
1159 : poBaseBuilder =
1160 120 : std::make_shared<arrow::Int32Builder>(m_poMemoryPool);
1161 288 : builder = std::make_shared<arrow::ListBuilder>(m_poMemoryPool,
1162 144 : poBaseBuilder);
1163 144 : break;
1164 : }
1165 :
1166 60 : case OFTInteger64List:
1167 60 : builder = std::make_shared<arrow::ListBuilder>(
1168 60 : m_poMemoryPool,
1169 180 : std::make_shared<arrow::Int64Builder>(m_poMemoryPool));
1170 :
1171 60 : break;
1172 :
1173 105 : case OFTRealList:
1174 105 : if (eSubDT == OFSTFloat32)
1175 33 : builder = std::make_shared<arrow::ListBuilder>(
1176 33 : m_poMemoryPool,
1177 99 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1178 : else
1179 72 : builder = std::make_shared<arrow::ListBuilder>(
1180 72 : m_poMemoryPool,
1181 216 : std::make_shared<arrow::DoubleBuilder>(m_poMemoryPool));
1182 105 : break;
1183 :
1184 36 : case OFTStringList:
1185 : case OFTWideStringList:
1186 36 : builder = std::make_shared<arrow::ListBuilder>(
1187 36 : m_poMemoryPool,
1188 108 : std::make_shared<arrow::StringBuilder>(m_poMemoryPool));
1189 :
1190 36 : break;
1191 :
1192 47 : case OFTDate:
1193 : builder =
1194 47 : std::make_shared<arrow::Date32Builder>(m_poMemoryPool);
1195 47 : break;
1196 :
1197 24 : case OFTTime:
1198 48 : builder = std::make_shared<arrow::Time32Builder>(
1199 72 : arrow::time32(arrow::TimeUnit::MILLI), m_poMemoryPool);
1200 24 : break;
1201 :
1202 95 : case OFTDateTime:
1203 190 : builder = std::make_shared<arrow::TimestampBuilder>(
1204 285 : arrow::timestamp(arrow::TimeUnit::MILLI), m_poMemoryPool);
1205 95 : break;
1206 : }
1207 1472 : m_apoBuilders.emplace_back(builder);
1208 : }
1209 :
1210 929 : for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i, ++nArrowIdx)
1211 : {
1212 460 : std::shared_ptr<arrow::ArrayBuilder> builder;
1213 460 : const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
1214 460 : const auto eGType = poGeomFieldDefn->GetType();
1215 : const int nDim =
1216 460 : 2 + (OGR_GT_HasZ(eGType) ? 1 : 0) + (OGR_GT_HasM(eGType) ? 1 : 0);
1217 :
1218 460 : switch (m_aeGeomEncoding[i])
1219 : {
1220 144 : case OGRArrowGeomEncoding::WKB:
1221 : builder =
1222 144 : std::make_shared<arrow::BinaryBuilder>(m_poMemoryPool);
1223 144 : break;
1224 :
1225 53 : case OGRArrowGeomEncoding::WKT:
1226 : builder =
1227 53 : std::make_shared<arrow::StringBuilder>(m_poMemoryPool);
1228 53 : break;
1229 :
1230 8 : case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
1231 8 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 0);
1232 8 : break;
1233 :
1234 8 : case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
1235 8 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 1);
1236 8 : break;
1237 :
1238 10 : case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
1239 10 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 2);
1240 10 : break;
1241 :
1242 8 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
1243 8 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 1);
1244 8 : break;
1245 :
1246 8 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
1247 8 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 2);
1248 8 : break;
1249 :
1250 10 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
1251 10 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 3);
1252 10 : break;
1253 :
1254 42 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
1255 84 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 0,
1256 84 : m_apoBaseStructGeomType[i]);
1257 42 : break;
1258 :
1259 29 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
1260 58 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 1,
1261 58 : m_apoBaseStructGeomType[i]);
1262 29 : break;
1263 :
1264 41 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
1265 82 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 2,
1266 82 : m_apoBaseStructGeomType[i]);
1267 41 : break;
1268 :
1269 29 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
1270 58 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 1,
1271 58 : m_apoBaseStructGeomType[i]);
1272 29 : break;
1273 :
1274 29 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
1275 58 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 2,
1276 58 : m_apoBaseStructGeomType[i]);
1277 29 : break;
1278 :
1279 41 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
1280 82 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 3,
1281 82 : m_apoBaseStructGeomType[i]);
1282 41 : break;
1283 :
1284 0 : case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
1285 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
1286 0 : CPLAssert(false);
1287 : break;
1288 : }
1289 :
1290 460 : m_apoBuilders.emplace_back(builder);
1291 :
1292 460 : if (m_bWriteBBoxStruct)
1293 : {
1294 : m_apoBuildersBBOXXMin.emplace_back(
1295 263 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1296 : m_apoBuildersBBOXYMin.emplace_back(
1297 263 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1298 : m_apoBuildersBBOXXMax.emplace_back(
1299 263 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1300 : m_apoBuildersBBOXYMax.emplace_back(
1301 263 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1302 : m_apoBuildersBBOXStruct.emplace_back(
1303 526 : std::make_shared<arrow::StructBuilder>(
1304 263 : m_apoFieldsBBOX[i]->type(), m_poMemoryPool,
1305 2367 : std::vector<std::shared_ptr<arrow::ArrayBuilder>>{
1306 263 : m_apoBuildersBBOXXMin.back(),
1307 263 : m_apoBuildersBBOXYMin.back(),
1308 263 : m_apoBuildersBBOXXMax.back(),
1309 1841 : m_apoBuildersBBOXYMax.back()}));
1310 : }
1311 : }
1312 469 : }
1313 :
1314 : /************************************************************************/
1315 : /* castToFloatDown() */
1316 : /************************************************************************/
1317 :
1318 : // Cf https://github.com/sqlite/sqlite/blob/90e4a3b7fcdf63035d6f35eb44d11ff58ff4b068/ext/rtree/rtree.c#L2993C1-L2995C3
1319 : /*
1320 : ** Rounding constants for float->double conversion.
1321 : */
1322 : #define RNDTOWARDS (1.0 - 1.0 / 8388608.0) /* Round towards zero */
1323 : #define RNDAWAY (1.0 + 1.0 / 8388608.0) /* Round away from zero */
1324 :
1325 : /*
1326 : ** Convert an sqlite3_value into an RtreeValue (presumably a float)
1327 : ** while taking care to round toward negative or positive, respectively.
1328 : */
1329 3820 : static float castToFloatDown(double d)
1330 : {
1331 3820 : float f = static_cast<float>(d);
1332 3820 : if (f > d)
1333 : {
1334 12 : f = static_cast<float>(d * (d < 0 ? RNDAWAY : RNDTOWARDS));
1335 : }
1336 3820 : return f;
1337 : }
1338 :
1339 3820 : static float castToFloatUp(double d)
1340 : {
1341 3820 : float f = static_cast<float>(d);
1342 3820 : if (f < d)
1343 : {
1344 7 : f = static_cast<float>(d * (d < 0 ? RNDTOWARDS : RNDAWAY));
1345 : }
1346 3820 : return f;
1347 : }
1348 :
1349 : /************************************************************************/
1350 : /* GeoArrowLineBuilder() */
1351 : /************************************************************************/
1352 :
1353 : template <class PointBuilderType>
1354 676 : static OGRErr GeoArrowLineBuilder(const OGRLineString *poLS,
1355 : PointBuilderType *poPointBuilder,
1356 : arrow::DoubleBuilder *poXBuilder,
1357 : arrow::DoubleBuilder *poYBuilder,
1358 : arrow::DoubleBuilder *poZBuilder,
1359 : arrow::DoubleBuilder *poMBuilder)
1360 : {
1361 3076 : for (int j = 0; j < poLS->getNumPoints(); ++j)
1362 : {
1363 2400 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1364 2400 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poXBuilder->Append(poLS->getX(j)));
1365 2400 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poYBuilder->Append(poLS->getY(j)));
1366 2400 : if (poZBuilder)
1367 668 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poZBuilder->Append(poLS->getZ(j)));
1368 2400 : if (poMBuilder)
1369 220 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMBuilder->Append(poLS->getM(j)));
1370 : }
1371 676 : return OGRERR_NONE;
1372 : }
1373 :
1374 : /************************************************************************/
1375 : /* BuildGeometry() */
1376 : /************************************************************************/
1377 :
1378 3876 : inline OGRErr OGRArrowWriterLayer::BuildGeometry(OGRGeometry *poGeom,
1379 : int iGeomField,
1380 : arrow::ArrayBuilder *poBuilder)
1381 : {
1382 3876 : const auto eGType = poGeom ? poGeom->getGeometryType() : wkbNone;
1383 : const auto eColumnGType =
1384 3876 : m_poFeatureDefn->GetGeomFieldDefn(iGeomField)->GetType();
1385 3876 : const bool bHasZ = CPL_TO_BOOL(OGR_GT_HasZ(eColumnGType));
1386 3876 : const bool bHasM = CPL_TO_BOOL(OGR_GT_HasM(eColumnGType));
1387 3876 : const bool bIsEmpty = poGeom != nullptr && poGeom->IsEmpty();
1388 3876 : OGREnvelope3D oEnvelope;
1389 3876 : if (poGeom != nullptr && !bIsEmpty)
1390 : {
1391 2258 : if (poGeom->Is3D())
1392 : {
1393 322 : poGeom->getEnvelope(&oEnvelope);
1394 322 : m_aoEnvelopes[iGeomField].Merge(oEnvelope);
1395 : }
1396 : else
1397 : {
1398 1936 : poGeom->getEnvelope(static_cast<OGREnvelope *>(&oEnvelope));
1399 1936 : m_aoEnvelopes[iGeomField].Merge(oEnvelope);
1400 : }
1401 2258 : m_oSetWrittenGeometryTypes[iGeomField].insert(eGType);
1402 : }
1403 :
1404 3876 : if (m_bWriteBBoxStruct)
1405 : {
1406 3173 : if (poGeom && !bIsEmpty)
1407 : {
1408 1867 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1409 : m_apoBuildersBBOXXMin[iGeomField]->Append(
1410 : castToFloatDown(oEnvelope.MinX)));
1411 1867 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1412 : m_apoBuildersBBOXYMin[iGeomField]->Append(
1413 : castToFloatDown(oEnvelope.MinY)));
1414 1867 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1415 : m_apoBuildersBBOXXMax[iGeomField]->Append(
1416 : castToFloatUp(oEnvelope.MaxX)));
1417 1867 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1418 : m_apoBuildersBBOXYMax[iGeomField]->Append(
1419 : castToFloatUp(oEnvelope.MaxY)));
1420 1867 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1421 : m_apoBuildersBBOXStruct[iGeomField]->Append());
1422 : }
1423 : else
1424 : {
1425 1306 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1426 : m_apoBuildersBBOXStruct[iGeomField]->AppendNull());
1427 : }
1428 : }
1429 :
1430 3876 : if (poGeom == nullptr)
1431 : {
1432 4020 : if (m_aeGeomEncoding[iGeomField] ==
1433 1348 : OGRArrowGeomEncoding::GEOARROW_FSL_POINT &&
1434 1348 : GetDriverUCName() == "PARQUET")
1435 : {
1436 : // For some reason, Parquet doesn't support a NULL FixedSizeList
1437 : // on reading
1438 4 : auto poPointBuilder =
1439 : static_cast<arrow::FixedSizeListBuilder *>(poBuilder);
1440 4 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1441 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1442 4 : poPointBuilder->value_builder());
1443 4 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1444 : std::numeric_limits<double>::quiet_NaN()));
1445 4 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1446 : std::numeric_limits<double>::quiet_NaN()));
1447 4 : if (bHasZ)
1448 2 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1449 : std::numeric_limits<double>::quiet_NaN()));
1450 4 : if (bHasM)
1451 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1452 : std::numeric_limits<double>::quiet_NaN()));
1453 : }
1454 : else
1455 : {
1456 1336 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1457 : }
1458 :
1459 1340 : return OGRERR_NONE;
1460 : }
1461 :
1462 : // The following checks are only valid for GeoArrow encoding
1463 3714 : if (m_aeGeomEncoding[iGeomField] != OGRArrowGeomEncoding::WKB &&
1464 1178 : m_aeGeomEncoding[iGeomField] != OGRArrowGeomEncoding::WKT)
1465 : {
1466 1114 : if ((!bIsEmpty && eGType != eColumnGType) ||
1467 244 : (bIsEmpty && wkbFlatten(eGType) != wkbFlatten(eColumnGType)))
1468 : {
1469 6 : CPLError(CE_Warning, CPLE_AppDefined,
1470 : "Geometry of type %s found, whereas %s is expected. "
1471 : "Writing null geometry",
1472 : OGRGeometryTypeToName(eGType),
1473 : OGRGeometryTypeToName(eColumnGType));
1474 6 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1475 :
1476 6 : return OGRERR_NONE;
1477 : }
1478 : }
1479 :
1480 2530 : switch (m_aeGeomEncoding[iGeomField])
1481 : {
1482 1358 : case OGRArrowGeomEncoding::WKB:
1483 : {
1484 0 : std::unique_ptr<OGRGeometry> poGeomModified;
1485 1358 : if (OGR_GT_HasM(eGType) && !OGR_GT_HasM(eColumnGType))
1486 : {
1487 : static bool bHasWarned = false;
1488 0 : if (!bHasWarned)
1489 : {
1490 0 : CPLError(CE_Warning, CPLE_AppDefined,
1491 : "Removing M component from geometry");
1492 0 : bHasWarned = true;
1493 : }
1494 0 : poGeomModified.reset(poGeom->clone());
1495 0 : poGeomModified->setMeasured(false);
1496 0 : poGeom = poGeomModified.get();
1497 : }
1498 1358 : FixupGeometryBeforeWriting(poGeom);
1499 1358 : const auto nSize = poGeom->WkbSize();
1500 1358 : if (nSize < INT_MAX)
1501 : {
1502 1358 : m_abyBuffer.resize(nSize);
1503 1358 : poGeom->exportToWkb(wkbNDR, &m_abyBuffer[0], wkbVariantIso);
1504 1358 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1505 : static_cast<arrow::BinaryBuilder *>(poBuilder)->Append(
1506 : m_abyBuffer.data(),
1507 : static_cast<int>(m_abyBuffer.size())));
1508 : }
1509 : else
1510 : {
1511 0 : CPLError(CE_Warning, CPLE_AppDefined,
1512 : "Too big geometry. "
1513 : "Writing null geometry");
1514 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1515 : }
1516 1358 : break;
1517 : }
1518 :
1519 308 : case OGRArrowGeomEncoding::WKT:
1520 : {
1521 308 : OGRWktOptions options;
1522 308 : options.variant = wkbVariantIso;
1523 308 : if (m_nWKTCoordinatePrecision >= 0)
1524 : {
1525 0 : options.format = OGRWktFormat::F;
1526 0 : options.xyPrecision = m_nWKTCoordinatePrecision;
1527 0 : options.zPrecision = m_nWKTCoordinatePrecision;
1528 0 : options.mPrecision = m_nWKTCoordinatePrecision;
1529 : }
1530 308 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1531 : static_cast<arrow::StringBuilder *>(poBuilder)->Append(
1532 : poGeom->exportToWkt(options)));
1533 308 : break;
1534 : }
1535 :
1536 20 : case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
1537 : {
1538 20 : const auto poPoint = poGeom->toPoint();
1539 20 : auto poPointBuilder =
1540 : static_cast<arrow::FixedSizeListBuilder *>(poBuilder);
1541 20 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1542 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1543 20 : poPointBuilder->value_builder());
1544 20 : if (bIsEmpty)
1545 : {
1546 8 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1547 : std::numeric_limits<double>::quiet_NaN()));
1548 8 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1549 : std::numeric_limits<double>::quiet_NaN()));
1550 8 : if (bHasZ)
1551 4 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1552 : std::numeric_limits<double>::quiet_NaN()));
1553 8 : if (bHasM)
1554 2 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1555 : std::numeric_limits<double>::quiet_NaN()));
1556 : }
1557 : else
1558 : {
1559 12 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1560 : poValueBuilder->Append(poPoint->getX()));
1561 12 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1562 : poValueBuilder->Append(poPoint->getY()));
1563 12 : if (bHasZ)
1564 6 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1565 : poValueBuilder->Append(poPoint->getZ()));
1566 12 : if (bHasM)
1567 2 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1568 : poValueBuilder->Append(poPoint->getM()));
1569 : }
1570 20 : break;
1571 : }
1572 :
1573 : #define GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder) \
1574 : auto poXBuilder = \
1575 : static_cast<arrow::DoubleBuilder *>(poPointBuilder->field_builder(0)); \
1576 : auto poYBuilder = \
1577 : static_cast<arrow::DoubleBuilder *>(poPointBuilder->field_builder(1)); \
1578 : int iSubField = 2; \
1579 : arrow::DoubleBuilder *poZBuilder = nullptr; \
1580 : if (bHasZ) \
1581 : { \
1582 : poZBuilder = static_cast<arrow::DoubleBuilder *>( \
1583 : poPointBuilder->field_builder(iSubField)); \
1584 : ++iSubField; \
1585 : } \
1586 : arrow::DoubleBuilder *poMBuilder = nullptr; \
1587 : if (bHasM) \
1588 : { \
1589 : poMBuilder = static_cast<arrow::DoubleBuilder *>( \
1590 : poPointBuilder->field_builder(iSubField)); \
1591 : } \
1592 : do \
1593 : { \
1594 : } while (0)
1595 :
1596 109 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
1597 : {
1598 109 : const auto poPoint = poGeom->toPoint();
1599 109 : auto poPointBuilder =
1600 : static_cast<arrow::StructBuilder *>(poBuilder);
1601 109 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1602 109 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1603 :
1604 109 : if (bIsEmpty)
1605 : {
1606 28 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poXBuilder->Append(
1607 : std::numeric_limits<double>::quiet_NaN()));
1608 28 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poYBuilder->Append(
1609 : std::numeric_limits<double>::quiet_NaN()));
1610 : }
1611 : else
1612 : {
1613 81 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1614 : poXBuilder->Append(poPoint->getX()));
1615 81 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1616 : poYBuilder->Append(poPoint->getY()));
1617 : }
1618 109 : if (poZBuilder)
1619 : {
1620 40 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poZBuilder->Append(
1621 : bIsEmpty ? std::numeric_limits<double>::quiet_NaN()
1622 : : poPoint->getZ()));
1623 : }
1624 109 : if (poMBuilder)
1625 : {
1626 4 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMBuilder->Append(
1627 : bIsEmpty ? std::numeric_limits<double>::quiet_NaN()
1628 : : poPoint->getM()));
1629 : }
1630 109 : break;
1631 : }
1632 :
1633 20 : case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
1634 : {
1635 20 : const auto poLS = poGeom->toLineString();
1636 20 : auto poListBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1637 : auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1638 20 : poListBuilder->value_builder());
1639 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1640 20 : poPointBuilder->value_builder());
1641 :
1642 20 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
1643 20 : OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1644 : poLS, poPointBuilder, poValueBuilder, poValueBuilder,
1645 : bHasZ ? poValueBuilder : nullptr,
1646 : bHasM ? poValueBuilder : nullptr));
1647 20 : break;
1648 : }
1649 :
1650 81 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
1651 : {
1652 81 : const auto poLS = poGeom->toLineString();
1653 81 : auto poListBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1654 : auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1655 81 : poListBuilder->value_builder());
1656 81 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1657 :
1658 81 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
1659 81 : OGR_ARROW_PROPAGATE_OGRERR(
1660 : GeoArrowLineBuilder(poLS, poPointBuilder, poXBuilder,
1661 : poYBuilder, poZBuilder, poMBuilder));
1662 81 : break;
1663 : }
1664 :
1665 32 : case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
1666 : {
1667 32 : const auto poPolygon = poGeom->toPolygon();
1668 32 : auto poPolygonBuilder =
1669 : static_cast<arrow::ListBuilder *>(poBuilder);
1670 : auto poRingBuilder = static_cast<arrow::ListBuilder *>(
1671 32 : poPolygonBuilder->value_builder());
1672 : auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1673 32 : poRingBuilder->value_builder());
1674 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1675 32 : poPointBuilder->value_builder());
1676 32 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPolygonBuilder->Append());
1677 62 : for (const auto *poRing : *poPolygon)
1678 : {
1679 30 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poRingBuilder->Append());
1680 30 : OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1681 : poRing, poPointBuilder, poValueBuilder, poValueBuilder,
1682 : bHasZ ? poValueBuilder : nullptr,
1683 : bHasM ? poValueBuilder : nullptr));
1684 : }
1685 32 : break;
1686 : }
1687 :
1688 133 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
1689 : {
1690 133 : const auto poPolygon = poGeom->toPolygon();
1691 133 : auto poPolygonBuilder =
1692 : static_cast<arrow::ListBuilder *>(poBuilder);
1693 : auto poRingBuilder = static_cast<arrow::ListBuilder *>(
1694 133 : poPolygonBuilder->value_builder());
1695 : auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1696 133 : poRingBuilder->value_builder());
1697 133 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1698 :
1699 133 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPolygonBuilder->Append());
1700 254 : for (const auto *poRing : *poPolygon)
1701 : {
1702 121 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poRingBuilder->Append());
1703 121 : OGR_ARROW_PROPAGATE_OGRERR(
1704 : GeoArrowLineBuilder(poRing, poPointBuilder, poXBuilder,
1705 : poYBuilder, poZBuilder, poMBuilder));
1706 : }
1707 133 : break;
1708 : }
1709 :
1710 32 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
1711 : {
1712 32 : const auto poMultiPoint = poGeom->toMultiPoint();
1713 32 : auto poListBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1714 : auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1715 32 : poListBuilder->value_builder());
1716 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1717 32 : poPointBuilder->value_builder());
1718 32 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
1719 88 : for (const auto *poPoint : *poMultiPoint)
1720 : {
1721 56 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1722 56 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1723 : poValueBuilder->Append(poPoint->getX()));
1724 56 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1725 : poValueBuilder->Append(poPoint->getY()));
1726 56 : if (bHasZ)
1727 28 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1728 : poValueBuilder->Append(poPoint->getZ()));
1729 56 : if (bHasM)
1730 18 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1731 : poValueBuilder->Append(poPoint->getM()));
1732 : }
1733 32 : break;
1734 : }
1735 :
1736 113 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
1737 : {
1738 113 : const auto poMultiPoint = poGeom->toMultiPoint();
1739 113 : auto poListBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1740 : auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1741 113 : poListBuilder->value_builder());
1742 113 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1743 :
1744 113 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
1745 270 : for (const auto *poPoint : *poMultiPoint)
1746 : {
1747 157 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1748 157 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1749 : poXBuilder->Append(poPoint->getX()));
1750 157 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1751 : poYBuilder->Append(poPoint->getY()));
1752 157 : if (poZBuilder)
1753 78 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1754 : poZBuilder->Append(poPoint->getZ()));
1755 157 : if (poMBuilder)
1756 18 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1757 : poMBuilder->Append(poPoint->getM()));
1758 : }
1759 113 : break;
1760 : }
1761 :
1762 28 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
1763 : {
1764 28 : const auto poMLS = poGeom->toMultiLineString();
1765 28 : auto poMLSBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1766 : auto poLSBuilder = static_cast<arrow::ListBuilder *>(
1767 28 : poMLSBuilder->value_builder());
1768 : auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1769 28 : poLSBuilder->value_builder());
1770 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1771 28 : poPointBuilder->value_builder());
1772 28 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMLSBuilder->Append());
1773 60 : for (const auto *poLS : *poMLS)
1774 : {
1775 32 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poLSBuilder->Append());
1776 32 : OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1777 : poLS, poPointBuilder, poValueBuilder, poValueBuilder,
1778 : bHasZ ? poValueBuilder : nullptr,
1779 : bHasM ? poValueBuilder : nullptr));
1780 : }
1781 28 : break;
1782 : }
1783 :
1784 109 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
1785 : {
1786 109 : const auto poMLS = poGeom->toMultiLineString();
1787 109 : auto poMLSBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1788 : auto poLSBuilder = static_cast<arrow::ListBuilder *>(
1789 109 : poMLSBuilder->value_builder());
1790 : auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1791 109 : poLSBuilder->value_builder());
1792 109 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1793 :
1794 109 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMLSBuilder->Append());
1795 242 : for (const auto *poLS : *poMLS)
1796 : {
1797 133 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poLSBuilder->Append());
1798 133 : OGR_ARROW_PROPAGATE_OGRERR(
1799 : GeoArrowLineBuilder(poLS, poPointBuilder, poXBuilder,
1800 : poYBuilder, poZBuilder, poMBuilder));
1801 : }
1802 109 : break;
1803 : }
1804 :
1805 38 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
1806 : {
1807 38 : const auto poMPoly = poGeom->toMultiPolygon();
1808 38 : auto poMPolyBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1809 : auto poPolyBuilder = static_cast<arrow::ListBuilder *>(
1810 38 : poMPolyBuilder->value_builder());
1811 : auto poRingBuilder = static_cast<arrow::ListBuilder *>(
1812 38 : poPolyBuilder->value_builder());
1813 : auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1814 38 : poRingBuilder->value_builder());
1815 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1816 38 : poPointBuilder->value_builder());
1817 38 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMPolyBuilder->Append());
1818 82 : for (const auto *poPolygon : *poMPoly)
1819 : {
1820 44 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPolyBuilder->Append());
1821 98 : for (const auto *poRing : *poPolygon)
1822 : {
1823 54 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poRingBuilder->Append());
1824 54 : OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1825 : poRing, poPointBuilder, poValueBuilder, poValueBuilder,
1826 : bHasZ ? poValueBuilder : nullptr,
1827 : bHasM ? poValueBuilder : nullptr));
1828 : }
1829 : }
1830 38 : break;
1831 : }
1832 :
1833 149 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
1834 : {
1835 149 : const auto poMPoly = poGeom->toMultiPolygon();
1836 149 : auto poMPolyBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1837 : auto poPolyBuilder = static_cast<arrow::ListBuilder *>(
1838 149 : poMPolyBuilder->value_builder());
1839 : auto poRingBuilder = static_cast<arrow::ListBuilder *>(
1840 149 : poPolyBuilder->value_builder());
1841 : auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1842 149 : poRingBuilder->value_builder());
1843 149 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1844 :
1845 149 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMPolyBuilder->Append());
1846 314 : for (const auto *poPolygon : *poMPoly)
1847 : {
1848 165 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPolyBuilder->Append());
1849 370 : for (const auto *poRing : *poPolygon)
1850 : {
1851 205 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poRingBuilder->Append());
1852 205 : OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1853 : poRing, poPointBuilder, poXBuilder, poYBuilder,
1854 : poZBuilder, poMBuilder));
1855 : }
1856 : }
1857 149 : break;
1858 : }
1859 :
1860 0 : case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
1861 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
1862 : {
1863 0 : CPLAssert(false);
1864 : break;
1865 : }
1866 : }
1867 :
1868 2530 : return OGRERR_NONE;
1869 : }
1870 :
1871 : /************************************************************************/
1872 : /* ICreateFeature() */
1873 : /************************************************************************/
1874 :
1875 3425 : inline OGRErr OGRArrowWriterLayer::ICreateFeature(OGRFeature *poFeature)
1876 : {
1877 3425 : if (m_poSchema == nullptr)
1878 : {
1879 310 : CreateSchema();
1880 : }
1881 :
1882 3425 : if (m_apoBuilders.empty())
1883 : {
1884 348 : if (!m_apoFieldsFromArrowSchema.empty())
1885 : {
1886 0 : CPLError(CE_Failure, CPLE_NotSupported,
1887 : "ICreateFeature() cannot be used after "
1888 : "CreateFieldFromArrowSchema()");
1889 0 : return OGRERR_FAILURE;
1890 : }
1891 348 : CreateArrayBuilders();
1892 : }
1893 :
1894 : // First pass to check not-null constraints as Arrow doesn't seem
1895 : // to do that on the writing side. But such files can't be read.
1896 3425 : const int nFieldCount = m_poFeatureDefn->GetFieldCount();
1897 9000 : for (int i = 0; i < nFieldCount; ++i)
1898 : {
1899 5576 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
1900 5578 : if (!poFieldDefn->IsNullable() &&
1901 2 : !poFeature->IsFieldSetAndNotNullUnsafe(i))
1902 : {
1903 1 : CPLError(CE_Failure, CPLE_AppDefined,
1904 : "Null value found in non-nullable field %s",
1905 : poFieldDefn->GetNameRef());
1906 1 : return OGRERR_FAILURE;
1907 : }
1908 : }
1909 :
1910 3424 : const int nGeomFieldCount = m_poFeatureDefn->GetGeomFieldCount();
1911 6951 : for (int i = 0; i < nGeomFieldCount; ++i)
1912 : {
1913 3527 : const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
1914 3559 : if (!poGeomFieldDefn->IsNullable() &&
1915 32 : poFeature->GetGeomFieldRef(i) == nullptr)
1916 : {
1917 0 : CPLError(CE_Failure, CPLE_AppDefined,
1918 : "Null value found in non-nullable geometry field %s",
1919 : poGeomFieldDefn->GetNameRef());
1920 0 : return OGRERR_FAILURE;
1921 : }
1922 : }
1923 :
1924 : // Write FID, if FID column present
1925 3424 : int nArrowIdx = 0;
1926 3424 : if (!m_osFIDColumn.empty())
1927 : {
1928 2261 : int64_t nFID = poFeature->GetFID();
1929 2261 : if (nFID == OGRNullFID)
1930 : {
1931 37 : nFID = m_nFeatureCount;
1932 37 : poFeature->SetFID(nFID);
1933 : }
1934 : auto poBuilder =
1935 2261 : static_cast<arrow::Int64Builder *>(m_apoBuilders[0].get());
1936 2261 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->Append(nFID));
1937 2261 : nArrowIdx++;
1938 : }
1939 :
1940 : // Write attributes
1941 8999 : for (int i = 0; i < nFieldCount; ++i, ++nArrowIdx)
1942 : {
1943 5575 : auto poBuilder = m_apoBuilders[nArrowIdx].get();
1944 5575 : if (!poFeature->IsFieldSetAndNotNullUnsafe(i))
1945 : {
1946 1245 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1947 1245 : continue;
1948 : }
1949 :
1950 4330 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
1951 4330 : const auto eSubDT = poFieldDefn->GetSubType();
1952 4330 : switch (poFieldDefn->GetType())
1953 : {
1954 2469 : case OFTInteger:
1955 2469 : if (eSubDT == OFSTBoolean)
1956 16 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1957 : static_cast<arrow::BooleanBuilder *>(poBuilder)->Append(
1958 : poFeature->GetFieldAsIntegerUnsafe(i) != 0));
1959 2453 : else if (eSubDT == OFSTInt16)
1960 16 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1961 : static_cast<arrow::Int16Builder *>(poBuilder)->Append(
1962 : static_cast<int16_t>(
1963 : poFeature->GetFieldAsIntegerUnsafe(i))));
1964 : else
1965 2437 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1966 : static_cast<arrow::Int32Builder *>(poBuilder)->Append(
1967 : poFeature->GetFieldAsIntegerUnsafe(i)));
1968 2469 : break;
1969 :
1970 172 : case OFTInteger64:
1971 172 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1972 : static_cast<arrow::Int64Builder *>(poBuilder)->Append(
1973 : static_cast<int64_t>(
1974 : poFeature->GetFieldAsInteger64Unsafe(i))));
1975 172 : break;
1976 :
1977 236 : case OFTReal:
1978 : {
1979 236 : const auto arrowType = m_poSchema->fields()[nArrowIdx]->type();
1980 236 : const double dfVal = poFeature->GetFieldAsDoubleUnsafe(i);
1981 236 : if (arrowType->id() == arrow::Type::DECIMAL128)
1982 : {
1983 : auto res = arrow::Decimal128::FromReal(
1984 : dfVal, poFieldDefn->GetWidth(),
1985 62 : poFieldDefn->GetPrecision());
1986 62 : if (res.ok())
1987 : {
1988 62 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1989 : static_cast<arrow::Decimal128Builder *>(poBuilder)
1990 : ->Append(*res));
1991 : }
1992 : else
1993 : {
1994 0 : CPLError(CE_Warning, CPLE_AppDefined,
1995 : "Cannot parse %.18g as a %d.%d decimal", dfVal,
1996 : poFieldDefn->GetWidth(),
1997 : poFieldDefn->GetPrecision());
1998 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1999 : }
2000 : }
2001 174 : else if (arrowType->id() == arrow::Type::DECIMAL256)
2002 : {
2003 : auto res = arrow::Decimal256::FromReal(
2004 : dfVal, poFieldDefn->GetWidth(),
2005 0 : poFieldDefn->GetPrecision());
2006 0 : if (res.ok())
2007 : {
2008 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2009 : static_cast<arrow::Decimal256Builder *>(poBuilder)
2010 : ->Append(*res));
2011 : }
2012 : else
2013 : {
2014 0 : CPLError(CE_Warning, CPLE_AppDefined,
2015 : "Cannot parse %.18g as a %d.%d decimal", dfVal,
2016 : poFieldDefn->GetWidth(),
2017 : poFieldDefn->GetPrecision());
2018 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
2019 : }
2020 : }
2021 174 : else if (eSubDT == OFSTFloat32)
2022 : {
2023 28 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2024 : static_cast<arrow::FloatBuilder *>(poBuilder)->Append(
2025 : static_cast<float>(dfVal)));
2026 : }
2027 : else
2028 : {
2029 146 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2030 : static_cast<arrow::DoubleBuilder *>(poBuilder)->Append(
2031 : dfVal));
2032 : }
2033 236 : break;
2034 : }
2035 :
2036 545 : case OFTString:
2037 : case OFTWideString:
2038 545 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2039 : static_cast<arrow::StringBuilder *>(poBuilder)->Append(
2040 : poFeature->GetFieldAsStringUnsafe(i)));
2041 545 : break;
2042 :
2043 62 : case OFTBinary:
2044 : {
2045 62 : int nSize = 0;
2046 62 : const auto pData = poFeature->GetFieldAsBinary(i, &nSize);
2047 62 : if (poFieldDefn->GetWidth() != 0)
2048 : {
2049 20 : if (poFieldDefn->GetWidth() != nSize)
2050 : {
2051 0 : CPLError(
2052 : CE_Warning, CPLE_AppDefined,
2053 : "Cannot write field %s. Got %d bytes, expected %d",
2054 : poFieldDefn->GetNameRef(), nSize,
2055 : poFieldDefn->GetWidth());
2056 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
2057 : }
2058 : else
2059 : {
2060 20 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2061 : static_cast<arrow::FixedSizeBinaryBuilder *>(
2062 : poBuilder)
2063 : ->Append(pData));
2064 : }
2065 : }
2066 : else
2067 42 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2068 : static_cast<arrow::BinaryBuilder *>(poBuilder)->Append(
2069 : pData, nSize));
2070 62 : break;
2071 : }
2072 :
2073 216 : case OFTIntegerList:
2074 : {
2075 216 : auto poListBuilder =
2076 : static_cast<arrow::ListBuilder *>(poBuilder);
2077 216 : if (eSubDT == OFSTBoolean)
2078 : {
2079 36 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2080 : auto poValueBuilder = static_cast<arrow::BooleanBuilder *>(
2081 36 : poListBuilder->value_builder());
2082 36 : int nValues = 0;
2083 : const auto panValues =
2084 36 : poFeature->GetFieldAsIntegerList(i, &nValues);
2085 108 : for (int j = 0; j < nValues; ++j)
2086 72 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2087 : poValueBuilder->Append(panValues[j] != 0));
2088 : }
2089 180 : else if (eSubDT == OFSTInt16)
2090 : {
2091 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2092 : auto poValueBuilder = static_cast<arrow::Int16Builder *>(
2093 0 : poListBuilder->value_builder());
2094 0 : int nValues = 0;
2095 : const auto panValues =
2096 0 : poFeature->GetFieldAsIntegerList(i, &nValues);
2097 0 : for (int j = 0; j < nValues; ++j)
2098 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
2099 : static_cast<int16_t>(panValues[j])));
2100 : }
2101 : else
2102 : {
2103 180 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2104 : auto poValueBuilder = static_cast<arrow::Int32Builder *>(
2105 180 : poListBuilder->value_builder());
2106 180 : int nValues = 0;
2107 : const auto panValues =
2108 180 : poFeature->GetFieldAsIntegerList(i, &nValues);
2109 540 : for (int j = 0; j < nValues; ++j)
2110 360 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2111 : poValueBuilder->Append(panValues[j]));
2112 : }
2113 216 : break;
2114 : }
2115 :
2116 92 : case OFTInteger64List:
2117 : {
2118 92 : auto poListBuilder =
2119 : static_cast<arrow::ListBuilder *>(poBuilder);
2120 92 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2121 : auto poValueBuilder = static_cast<arrow::Int64Builder *>(
2122 92 : poListBuilder->value_builder());
2123 92 : int nValues = 0;
2124 : const auto panValues =
2125 92 : poFeature->GetFieldAsInteger64List(i, &nValues);
2126 292 : for (int j = 0; j < nValues; ++j)
2127 200 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
2128 : static_cast<int64_t>(panValues[j])));
2129 92 : break;
2130 : }
2131 :
2132 152 : case OFTRealList:
2133 : {
2134 152 : auto poListBuilder =
2135 : static_cast<arrow::ListBuilder *>(poBuilder);
2136 152 : if (eSubDT == OFSTFloat32)
2137 : {
2138 48 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2139 : auto poValueBuilder = static_cast<arrow::FloatBuilder *>(
2140 48 : poListBuilder->value_builder());
2141 48 : int nValues = 0;
2142 : const auto padfValues =
2143 48 : poFeature->GetFieldAsDoubleList(i, &nValues);
2144 144 : for (int j = 0; j < nValues; ++j)
2145 96 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
2146 : static_cast<float>(padfValues[j])));
2147 : }
2148 : else
2149 : {
2150 104 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2151 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
2152 104 : poListBuilder->value_builder());
2153 104 : int nValues = 0;
2154 : const auto padfValues =
2155 104 : poFeature->GetFieldAsDoubleList(i, &nValues);
2156 280 : for (int j = 0; j < nValues; ++j)
2157 176 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2158 : poValueBuilder->Append(padfValues[j]));
2159 : }
2160 152 : break;
2161 : }
2162 :
2163 52 : case OFTStringList:
2164 : case OFTWideStringList:
2165 : {
2166 52 : auto poListBuilder =
2167 : static_cast<arrow::ListBuilder *>(poBuilder);
2168 52 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2169 : auto poValueBuilder = static_cast<arrow::StringBuilder *>(
2170 52 : poListBuilder->value_builder());
2171 52 : const auto papszValues = poFeature->GetFieldAsStringList(i);
2172 132 : for (int j = 0; papszValues && papszValues[j]; ++j)
2173 80 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2174 : poValueBuilder->Append(papszValues[j]));
2175 52 : break;
2176 : }
2177 :
2178 109 : case OFTDate:
2179 : {
2180 : int nYear, nMonth, nDay, nHour, nMinute;
2181 : float fSec;
2182 : int nTZFlag;
2183 109 : poFeature->GetFieldAsDateTime(i, &nYear, &nMonth, &nDay, &nHour,
2184 : &nMinute, &fSec, &nTZFlag);
2185 : struct tm brokenDown;
2186 109 : memset(&brokenDown, 0, sizeof(brokenDown));
2187 109 : brokenDown.tm_year = nYear - 1900;
2188 109 : brokenDown.tm_mon = nMonth - 1;
2189 109 : brokenDown.tm_mday = nDay;
2190 109 : GIntBig nVal = CPLYMDHMSToUnixTime(&brokenDown);
2191 109 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2192 : static_cast<arrow::Date32Builder *>(poBuilder)->Append(
2193 : static_cast<int>(nVal / 86400)));
2194 109 : break;
2195 : }
2196 :
2197 36 : case OFTTime:
2198 : {
2199 : int nYear, nMonth, nDay, nHour, nMinute;
2200 : float fSec;
2201 : int nTZFlag;
2202 36 : poFeature->GetFieldAsDateTime(i, &nYear, &nMonth, &nDay, &nHour,
2203 : &nMinute, &fSec, &nTZFlag);
2204 36 : int nVal = nHour * 3600 + nMinute * 60;
2205 36 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2206 : static_cast<arrow::Time32Builder *>(poBuilder)->Append(
2207 : static_cast<int>(
2208 : (static_cast<double>(nVal) + fSec) * 1000 + 0.5)));
2209 36 : break;
2210 : }
2211 :
2212 189 : case OFTDateTime:
2213 : {
2214 : int nYear, nMonth, nDay, nHour, nMinute;
2215 : float fSec;
2216 : int nTZFlag;
2217 189 : poFeature->GetFieldAsDateTime(i, &nYear, &nMonth, &nDay, &nHour,
2218 : &nMinute, &fSec, &nTZFlag);
2219 : struct tm brokenDown;
2220 189 : memset(&brokenDown, 0, sizeof(brokenDown));
2221 189 : brokenDown.tm_year = nYear - 1900;
2222 189 : brokenDown.tm_mon = nMonth - 1;
2223 189 : brokenDown.tm_mday = nDay;
2224 189 : brokenDown.tm_hour = nHour;
2225 189 : brokenDown.tm_min = nMinute;
2226 189 : brokenDown.tm_sec = 0;
2227 189 : GIntBig nVal = CPLYMDHMSToUnixTime(&brokenDown);
2228 306 : if (!IsFileWriterCreated() &&
2229 117 : m_anTZFlag[i] != OGR_TZFLAG_UNKNOWN)
2230 : {
2231 59 : if (m_anTZFlag[i] == TZFLAG_UNINITIALIZED)
2232 35 : m_anTZFlag[i] = nTZFlag;
2233 24 : else if (m_anTZFlag[i] != nTZFlag)
2234 : {
2235 0 : if (m_anTZFlag[i] >= OGR_TZFLAG_MIXED_TZ &&
2236 0 : nTZFlag >= OGR_TZFLAG_MIXED_TZ)
2237 : {
2238 0 : m_anTZFlag[i] =
2239 : OGR_TZFLAG_MIXED_TZ; // harmonize on UTC ultimately
2240 : }
2241 : else
2242 : {
2243 0 : CPLError(CE_Warning, CPLE_AppDefined,
2244 : "Field %s contains a mix of "
2245 : "timezone-aware and local/without "
2246 : "timezone values.",
2247 : poFieldDefn->GetNameRef());
2248 0 : m_anTZFlag[i] = OGR_TZFLAG_UNKNOWN;
2249 : }
2250 : }
2251 : }
2252 189 : if (nTZFlag > OGR_TZFLAG_MIXED_TZ)
2253 : {
2254 60 : const int nOffsetSec = (nTZFlag - OGR_TZFLAG_UTC) * 15 * 60;
2255 60 : nVal -= nOffsetSec;
2256 : }
2257 189 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2258 : static_cast<arrow::TimestampBuilder *>(poBuilder)->Append(
2259 : static_cast<int64_t>(
2260 : (static_cast<double>(nVal) + fSec) * 1000 + 0.5)));
2261 189 : break;
2262 : }
2263 : }
2264 : }
2265 :
2266 : // Write geometries
2267 6951 : for (int i = 0; i < nGeomFieldCount; ++i, ++nArrowIdx)
2268 : {
2269 3527 : auto poBuilder = m_apoBuilders[nArrowIdx].get();
2270 3527 : OGRGeometry *poGeom = poFeature->GetGeomFieldRef(i);
2271 3527 : if (BuildGeometry(poGeom, i, poBuilder) != OGRERR_NONE)
2272 0 : return OGRERR_FAILURE;
2273 : }
2274 :
2275 3424 : m_nFeatureCount++;
2276 :
2277 : // Flush the current row group if reaching the limit of rows per group.
2278 3424 : if (!m_apoBuilders.empty() && m_apoBuilders[0]->length() == m_nRowGroupSize)
2279 : {
2280 25 : if (!FlushFeatures())
2281 0 : return OGRERR_FAILURE;
2282 : }
2283 :
2284 3424 : return OGRERR_NONE;
2285 : }
2286 :
2287 : /************************************************************************/
2288 : /* FlushFeatures() */
2289 : /************************************************************************/
2290 :
2291 42 : inline bool OGRArrowWriterLayer::FlushFeatures()
2292 : {
2293 42 : if (m_apoBuilders.empty() || m_apoBuilders[0]->length() == 0)
2294 1 : return true;
2295 :
2296 41 : if (!IsFileWriterCreated())
2297 : {
2298 10 : CreateWriter();
2299 10 : if (!IsFileWriterCreated())
2300 0 : return false;
2301 : }
2302 :
2303 41 : return FlushGroup();
2304 : }
2305 :
2306 : /************************************************************************/
2307 : /* GetFeatureCount() */
2308 : /************************************************************************/
2309 :
2310 1 : inline GIntBig OGRArrowWriterLayer::GetFeatureCount(int bForce)
2311 : {
2312 1 : if (m_poAttrQuery == nullptr && m_poFilterGeom == nullptr)
2313 : {
2314 1 : return m_nFeatureCount;
2315 : }
2316 0 : return OGRLayer::GetFeatureCount(bForce);
2317 : }
2318 :
2319 : /************************************************************************/
2320 : /* TestCapability() */
2321 : /************************************************************************/
2322 :
2323 795 : inline int OGRArrowWriterLayer::TestCapability(const char *pszCap) const
2324 : {
2325 795 : if (EQUAL(pszCap, OLCCreateField) || EQUAL(pszCap, OLCCreateGeomField))
2326 27 : return m_poSchema == nullptr;
2327 :
2328 768 : if (EQUAL(pszCap, OLCSequentialWrite))
2329 24 : return true;
2330 :
2331 744 : if (EQUAL(pszCap, OLCFastWriteArrowBatch))
2332 0 : return true;
2333 :
2334 744 : if (EQUAL(pszCap, OLCStringsAsUTF8))
2335 1 : return true;
2336 :
2337 743 : if (EQUAL(pszCap, OLCMeasuredGeometries))
2338 310 : return true;
2339 :
2340 433 : return false;
2341 : }
2342 :
2343 : /************************************************************************/
2344 : /* WriteArrays() */
2345 : /************************************************************************/
2346 :
2347 350 : inline bool OGRArrowWriterLayer::WriteArrays(
2348 : std::function<bool(const std::shared_ptr<arrow::Field> &,
2349 : const std::shared_ptr<arrow::Array> &)>
2350 : postProcessArray)
2351 : {
2352 350 : int nArrowIdx = 0;
2353 350 : int nArrowIdxFirstField = !m_osFIDColumn.empty() ? 1 : 0;
2354 2213 : for (const auto &poBuilder : m_apoBuilders)
2355 : {
2356 1863 : const auto &field = m_poSchema->fields()[nArrowIdx];
2357 :
2358 0 : std::shared_ptr<arrow::Array> array;
2359 1863 : auto status = poBuilder->Finish(&array);
2360 1863 : if (!status.ok())
2361 : {
2362 0 : CPLError(CE_Failure, CPLE_AppDefined,
2363 : "builder::Finish() for field %s failed with %s",
2364 0 : field->name().c_str(), status.message().c_str());
2365 0 : return false;
2366 : }
2367 :
2368 : // CPLDebug("ARROW", "%s", array->ToString().c_str());
2369 :
2370 1863 : const int iCol = nArrowIdx - nArrowIdxFirstField;
2371 1863 : if (iCol >= 0 && iCol < m_poFeatureDefn->GetFieldCount())
2372 : {
2373 1472 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(iCol);
2374 1472 : const auto eFieldType = poFieldDefn->GetType();
2375 1472 : if (eFieldType == OFTInteger || eFieldType == OFTInteger64)
2376 : {
2377 256 : const auto &osDomainName = poFieldDefn->GetDomainName();
2378 : const auto oIter =
2379 256 : m_oMapFieldDomainToStringArray.find(osDomainName);
2380 256 : if (oIter != m_oMapFieldDomainToStringArray.end())
2381 : {
2382 : auto result = arrow::DictionaryArray::FromArrays(
2383 12 : field->type(), array, oIter->second);
2384 12 : if (!result.ok())
2385 : {
2386 0 : CPLError(CE_Failure, CPLE_AppDefined,
2387 : "DictionaryArray::FromArrays() for field %s "
2388 : "failed with %s",
2389 0 : field->name().c_str(),
2390 0 : result.status().message().c_str());
2391 0 : return false;
2392 : }
2393 12 : array = *result;
2394 : }
2395 : }
2396 : }
2397 :
2398 1863 : if (!postProcessArray(field, array))
2399 : {
2400 0 : return false;
2401 : }
2402 :
2403 1863 : nArrowIdx++;
2404 : }
2405 :
2406 350 : if (m_bWriteBBoxStruct)
2407 : {
2408 260 : const int nGeomFieldCount = m_poFeatureDefn->GetGeomFieldCount();
2409 516 : for (int i = 0; i < nGeomFieldCount; ++i)
2410 : {
2411 256 : const auto &field = m_apoFieldsBBOX[i];
2412 0 : std::shared_ptr<arrow::Array> array;
2413 256 : auto status = m_apoBuildersBBOXStruct[i]->Finish(&array);
2414 256 : if (!status.ok())
2415 : {
2416 0 : CPLError(CE_Failure, CPLE_AppDefined,
2417 : "builder::Finish() for field %s failed with %s",
2418 0 : field->name().c_str(), status.message().c_str());
2419 0 : return false;
2420 : }
2421 :
2422 256 : if (!postProcessArray(field, array))
2423 : {
2424 0 : return false;
2425 : }
2426 : }
2427 : }
2428 :
2429 350 : return true;
2430 : }
2431 :
2432 : /************************************************************************/
2433 : /* TestBit() */
2434 : /************************************************************************/
2435 :
2436 504 : static inline bool TestBit(const uint8_t *pabyData, size_t nIdx)
2437 : {
2438 504 : return (pabyData[nIdx / 8] & (1 << (nIdx % 8))) != 0;
2439 : }
2440 :
2441 : /************************************************************************/
2442 : /* WriteArrowBatchInternal() */
2443 : /************************************************************************/
2444 :
2445 126 : inline bool OGRArrowWriterLayer::WriteArrowBatchInternal(
2446 : const struct ArrowSchema *schema, struct ArrowArray *array,
2447 : CSLConstList papszOptions,
2448 : std::function<bool(const std::shared_ptr<arrow::RecordBatch> &)> writeBatch)
2449 : {
2450 : #ifdef __COVERITY__
2451 : (void)schema;
2452 : (void)array;
2453 : (void)papszOptions;
2454 : (void)writeBatch;
2455 : CPLError(CE_Failure, CPLE_AppDefined, "Not implemented");
2456 : return false;
2457 : #else
2458 126 : if (m_poSchema == nullptr)
2459 : {
2460 121 : CreateSchema();
2461 : }
2462 :
2463 126 : if (!IsFileWriterCreated())
2464 : {
2465 121 : CreateWriter();
2466 121 : if (!IsFileWriterCreated())
2467 0 : return false;
2468 : }
2469 :
2470 126 : if (m_apoBuilders.empty())
2471 : {
2472 121 : CreateArrayBuilders();
2473 : }
2474 :
2475 126 : const int nGeomFieldCount = m_poFeatureDefn->GetGeomFieldCount();
2476 126 : const int nGeomFieldCountBBoxFields =
2477 126 : m_bWriteBBoxStruct ? nGeomFieldCount : 0;
2478 :
2479 126 : const char *pszFIDName = CSLFetchNameValueDef(
2480 : papszOptions, "FID", OGRLayer::DEFAULT_ARROW_FID_NAME);
2481 : const char *pszSingleGeomFieldName =
2482 126 : CSLFetchNameValue(papszOptions, "GEOMETRY_NAME");
2483 :
2484 : // Sort schema and array children in the same order as m_poSchema.
2485 : // This is needed for non-WKB geometry encoding
2486 252 : std::map<std::string, int> oMapSchemaChildrenNameToIdx;
2487 1644 : for (int i = 0; i < static_cast<int>(schema->n_children); ++i)
2488 : {
2489 1518 : if (cpl::contains(oMapSchemaChildrenNameToIdx,
2490 1518 : schema->children[i]->name))
2491 : {
2492 0 : CPLError(CE_Failure, CPLE_AppDefined,
2493 : "Several fields with same name '%s' found",
2494 0 : schema->children[i]->name);
2495 0 : return false;
2496 : }
2497 1518 : oMapSchemaChildrenNameToIdx[schema->children[i]->name] = i;
2498 :
2499 1518 : if (!pszSingleGeomFieldName && schema->children[i]->metadata)
2500 : {
2501 : const auto oMetadata =
2502 256 : OGRParseArrowMetadata(schema->children[i]->metadata);
2503 128 : const auto oIter = oMetadata.find(ARROW_EXTENSION_NAME_KEY);
2504 262 : if (oIter != oMetadata.end() &&
2505 134 : (oIter->second == EXTENSION_NAME_OGC_WKB ||
2506 8 : oIter->second == EXTENSION_NAME_GEOARROW_WKB))
2507 : {
2508 126 : pszSingleGeomFieldName = schema->children[i]->name;
2509 : }
2510 : }
2511 : }
2512 126 : if (!pszSingleGeomFieldName)
2513 0 : pszSingleGeomFieldName = OGRLayer::DEFAULT_ARROW_GEOMETRY_NAME;
2514 :
2515 126 : std::vector<int> anMapLayerSchemaToArraySchema(m_poSchema->num_fields(),
2516 378 : -1);
2517 : struct ArrowArray fidArray;
2518 : struct ArrowSchema fidSchema;
2519 126 : memset(&fidArray, 0, sizeof(fidArray));
2520 126 : memset(&fidSchema, 0, sizeof(fidSchema));
2521 252 : std::vector<void *> apBuffersFid;
2522 252 : std::vector<int64_t> fids;
2523 :
2524 252 : std::set<int> oSetReferencedFieldsInArraySchema;
2525 0 : const auto DummyFreeArray = [](struct ArrowArray *ptrArray)
2526 0 : { ptrArray->release = nullptr; };
2527 126 : const auto DummyFreeSchema = [](struct ArrowSchema *ptrSchema)
2528 126 : { ptrSchema->release = nullptr; };
2529 126 : bool bRebuildBatch = false;
2530 1642 : for (int i = 0; i < m_poSchema->num_fields() - nGeomFieldCountBBoxFields;
2531 : ++i)
2532 : {
2533 : auto oIter =
2534 1516 : oMapSchemaChildrenNameToIdx.find(m_poSchema->field(i)->name());
2535 1516 : if (oIter == oMapSchemaChildrenNameToIdx.end())
2536 : {
2537 4 : if (m_poSchema->field(i)->name() == m_osFIDColumn)
2538 : {
2539 1 : oIter = oMapSchemaChildrenNameToIdx.find(pszFIDName);
2540 1 : if (oIter == oMapSchemaChildrenNameToIdx.end())
2541 : {
2542 : // If the input data does not contain a FID column, but
2543 : // the output file requires it, creates a default FID column
2544 0 : fidArray.release = DummyFreeArray;
2545 0 : fidArray.n_buffers = 2;
2546 0 : apBuffersFid.resize(2);
2547 0 : fidArray.buffers =
2548 0 : const_cast<const void **>(apBuffersFid.data());
2549 0 : fids.reserve(static_cast<size_t>(array->length));
2550 0 : for (size_t iRow = 0;
2551 0 : iRow < static_cast<size_t>(array->length); ++iRow)
2552 0 : fids.push_back(m_nFeatureCount + iRow);
2553 0 : fidArray.buffers[1] = fids.data();
2554 0 : fidArray.length = array->length;
2555 0 : fidSchema.release = DummyFreeSchema;
2556 0 : fidSchema.name = m_osFIDColumn.c_str();
2557 0 : fidSchema.format = "l"; // int64
2558 0 : continue;
2559 : }
2560 : }
2561 6 : else if (nGeomFieldCount == 1 &&
2562 3 : m_poFeatureDefn->GetGeomFieldIndex(
2563 3 : m_poSchema->field(i)->name().c_str()) == 0)
2564 : {
2565 : oIter =
2566 3 : oMapSchemaChildrenNameToIdx.find(pszSingleGeomFieldName);
2567 3 : if (oIter != oMapSchemaChildrenNameToIdx.end())
2568 3 : bRebuildBatch = true;
2569 : }
2570 :
2571 4 : if (oIter == oMapSchemaChildrenNameToIdx.end())
2572 : {
2573 0 : CPLError(CE_Failure, CPLE_AppDefined,
2574 : "Cannot find field '%s' in schema",
2575 0 : m_poSchema->field(i)->name().c_str());
2576 0 : return false;
2577 : }
2578 : }
2579 1516 : anMapLayerSchemaToArraySchema[i] = oIter->second;
2580 1516 : oSetReferencedFieldsInArraySchema.insert(oIter->second);
2581 : }
2582 :
2583 : // Note: we cheat a bit by declaring a single instance of the minx/miny/
2584 : // maxx/maxy sub-field ArrowSchema*, and make all struct ArrowSchema point
2585 : // to them. That's OK because we use DummyFreeSchema to release, which does
2586 : // nothing.
2587 : struct ArrowSchema bboxStructSchemaXMin;
2588 : struct ArrowSchema bboxStructSchemaYMin;
2589 : struct ArrowSchema bboxStructSchemaXMax;
2590 : struct ArrowSchema bboxStructSchemaYMax;
2591 126 : constexpr int BBOX_SUBFIELD_COUNT = 4;
2592 : std::array<struct ArrowSchema *, BBOX_SUBFIELD_COUNT>
2593 : bboxStructSchemaChildren;
2594 126 : constexpr int BBOX_STRUCT_BUFFER_COUNT = 1; // validity bitmap array
2595 : // cppcheck-suppress constStatement
2596 : std::vector<std::array<const void *, BBOX_STRUCT_BUFFER_COUNT>>
2597 252 : bboxStructBuffersPtr;
2598 252 : std::vector<std::vector<GByte>> aabyBboxStructValidity;
2599 252 : std::vector<std::vector<float>> aadfMinX, aadfMinY, aadfMaxX, aadfMaxY;
2600 : // cppcheck-suppress constStatement
2601 252 : std::vector<std::array<struct ArrowArray, BBOX_SUBFIELD_COUNT>> bboxArrays;
2602 : // cppcheck-suppress constStatement
2603 : std::vector<std::array<struct ArrowArray *, BBOX_SUBFIELD_COUNT>>
2604 252 : bboxArraysPtr;
2605 126 : constexpr int BBOX_SUBFIELD_BUFFER_COUNT =
2606 : 2; // validity bitmap array and float array
2607 : std::vector<std::array<std::array<const void *, BBOX_SUBFIELD_BUFFER_COUNT>,
2608 : BBOX_SUBFIELD_COUNT>>
2609 252 : bboxBuffersPtr;
2610 :
2611 : // Temporary arrays to hold the geometry bounding boxes.
2612 252 : std::vector<struct ArrowArray> bboxStructArray;
2613 252 : std::vector<struct ArrowSchema> bboxStructSchema;
2614 :
2615 252 : std::vector<struct ArrowSchema *> newSchemaChildren;
2616 252 : std::vector<struct ArrowArray *> newArrayChildren;
2617 126 : newSchemaChildren.reserve(m_poSchema->num_fields());
2618 126 : newArrayChildren.reserve(m_poSchema->num_fields());
2619 1642 : for (int i = 0; i < m_poSchema->num_fields() - nGeomFieldCountBBoxFields;
2620 : ++i)
2621 : {
2622 1516 : if (anMapLayerSchemaToArraySchema[i] < 0)
2623 : {
2624 0 : CPLAssert(m_poSchema->field(i)->name() == m_osFIDColumn);
2625 0 : newSchemaChildren.emplace_back(&fidSchema);
2626 0 : newArrayChildren.emplace_back(&fidArray);
2627 : }
2628 : else
2629 : {
2630 : newSchemaChildren.emplace_back(
2631 1516 : schema->children[anMapLayerSchemaToArraySchema[i]]);
2632 : newArrayChildren.emplace_back(
2633 1516 : array->children[anMapLayerSchemaToArraySchema[i]]);
2634 : }
2635 : }
2636 :
2637 126 : if (m_bWriteBBoxStruct)
2638 : {
2639 14 : memset(&bboxStructSchemaXMin, 0, sizeof(bboxStructSchemaXMin));
2640 14 : memset(&bboxStructSchemaYMin, 0, sizeof(bboxStructSchemaYMin));
2641 14 : memset(&bboxStructSchemaXMax, 0, sizeof(bboxStructSchemaXMax));
2642 14 : memset(&bboxStructSchemaYMax, 0, sizeof(bboxStructSchemaYMax));
2643 :
2644 14 : bboxStructSchemaXMin.release = DummyFreeSchema;
2645 14 : bboxStructSchemaXMin.name = "xmin";
2646 14 : bboxStructSchemaXMin.format = "f"; // float32
2647 :
2648 14 : bboxStructSchemaYMin.release = DummyFreeSchema;
2649 14 : bboxStructSchemaYMin.name = "ymin";
2650 14 : bboxStructSchemaYMin.format = "f"; // float32
2651 :
2652 14 : bboxStructSchemaXMax.release = DummyFreeSchema;
2653 14 : bboxStructSchemaXMax.name = "xmax";
2654 14 : bboxStructSchemaXMax.format = "f"; // float32
2655 :
2656 14 : bboxStructSchemaYMax.release = DummyFreeSchema;
2657 14 : bboxStructSchemaYMax.name = "ymax";
2658 14 : bboxStructSchemaYMax.format = "f"; // float32
2659 :
2660 : try
2661 : {
2662 14 : constexpr int XMIN_IDX = 0;
2663 14 : constexpr int YMIN_IDX = 1;
2664 14 : constexpr int XMAX_IDX = 2;
2665 14 : constexpr int YMAX_IDX = 3;
2666 14 : bboxStructSchemaChildren[XMIN_IDX] = &bboxStructSchemaXMin;
2667 : // cppcheck-suppress objectIndex
2668 14 : bboxStructSchemaChildren[YMIN_IDX] = &bboxStructSchemaYMin;
2669 : // cppcheck-suppress objectIndex
2670 14 : bboxStructSchemaChildren[XMAX_IDX] = &bboxStructSchemaXMax;
2671 : // cppcheck-suppress objectIndex
2672 14 : bboxStructSchemaChildren[YMAX_IDX] = &bboxStructSchemaYMax;
2673 :
2674 14 : bboxStructArray.resize(nGeomFieldCount);
2675 14 : bboxStructSchema.resize(nGeomFieldCount);
2676 14 : bboxArrays.resize(nGeomFieldCount);
2677 14 : bboxArraysPtr.resize(nGeomFieldCount);
2678 14 : bboxBuffersPtr.resize(nGeomFieldCount);
2679 14 : bboxStructBuffersPtr.resize(nGeomFieldCount);
2680 14 : aabyBboxStructValidity.resize(nGeomFieldCount);
2681 28 : memset(bboxStructArray.data(), 0,
2682 14 : nGeomFieldCount * sizeof(bboxStructArray[0]));
2683 28 : memset(bboxStructSchema.data(), 0,
2684 14 : nGeomFieldCount * sizeof(bboxStructSchema[0]));
2685 28 : memset(bboxArrays.data(), 0,
2686 14 : nGeomFieldCount * sizeof(bboxArrays[0]));
2687 14 : aadfMinX.resize(nGeomFieldCount);
2688 14 : aadfMinY.resize(nGeomFieldCount);
2689 14 : aadfMaxX.resize(nGeomFieldCount);
2690 14 : aadfMaxY.resize(nGeomFieldCount);
2691 28 : for (int i = 0; i < nGeomFieldCount; ++i)
2692 : {
2693 14 : const bool bIsNullable = CPL_TO_BOOL(
2694 14 : m_poFeatureDefn->GetGeomFieldDefn(i)->IsNullable());
2695 14 : aadfMinX[i].reserve(static_cast<size_t>(array->length));
2696 14 : aadfMinY[i].reserve(static_cast<size_t>(array->length));
2697 14 : aadfMaxX[i].reserve(static_cast<size_t>(array->length));
2698 14 : aadfMaxY[i].reserve(static_cast<size_t>(array->length));
2699 14 : aabyBboxStructValidity[i].resize(
2700 14 : static_cast<size_t>(array->length + 7) / 8, 0xFF);
2701 :
2702 14 : bboxStructSchema[i].release = DummyFreeSchema;
2703 14 : bboxStructSchema[i].name = m_apoFieldsBBOX[i]->name().c_str();
2704 14 : bboxStructSchema[i].format = "+s"; // structure
2705 14 : bboxStructSchema[i].flags =
2706 14 : bIsNullable ? ARROW_FLAG_NULLABLE : 0;
2707 14 : bboxStructSchema[i].n_children = BBOX_SUBFIELD_COUNT;
2708 14 : bboxStructSchema[i].children = bboxStructSchemaChildren.data();
2709 :
2710 14 : constexpr int VALIDITY_ARRAY_IDX = 0;
2711 14 : constexpr int BBOX_SUBFIELD_FLOAT_VALUE_IDX = 1;
2712 14 : bboxBuffersPtr[i][XMIN_IDX][BBOX_SUBFIELD_FLOAT_VALUE_IDX] =
2713 14 : aadfMinX[i].data();
2714 14 : bboxBuffersPtr[i][YMIN_IDX][BBOX_SUBFIELD_FLOAT_VALUE_IDX] =
2715 14 : aadfMinY[i].data();
2716 14 : bboxBuffersPtr[i][XMAX_IDX][BBOX_SUBFIELD_FLOAT_VALUE_IDX] =
2717 14 : aadfMaxX[i].data();
2718 14 : bboxBuffersPtr[i][YMAX_IDX][BBOX_SUBFIELD_FLOAT_VALUE_IDX] =
2719 14 : aadfMaxY[i].data();
2720 :
2721 70 : for (int j = 0; j < BBOX_SUBFIELD_COUNT; ++j)
2722 : {
2723 56 : bboxBuffersPtr[i][j][VALIDITY_ARRAY_IDX] = nullptr;
2724 :
2725 56 : bboxArrays[i][j].release = DummyFreeArray;
2726 56 : bboxArrays[i][j].length = array->length;
2727 56 : bboxArrays[i][j].n_buffers = BBOX_SUBFIELD_BUFFER_COUNT;
2728 56 : bboxArrays[i][j].buffers = bboxBuffersPtr[i][j].data();
2729 :
2730 56 : bboxArraysPtr[i][j] = &bboxArrays[i][j];
2731 : }
2732 :
2733 14 : bboxStructArray[i].release = DummyFreeArray;
2734 14 : bboxStructArray[i].n_children = BBOX_SUBFIELD_COUNT;
2735 : // coverity[escape]
2736 14 : bboxStructArray[i].children = bboxArraysPtr[i].data();
2737 14 : bboxStructArray[i].length = array->length;
2738 14 : bboxStructArray[i].n_buffers = BBOX_STRUCT_BUFFER_COUNT;
2739 14 : bboxStructBuffersPtr[i][VALIDITY_ARRAY_IDX] =
2740 14 : bIsNullable ? aabyBboxStructValidity[i].data() : nullptr;
2741 : // coverity[escape]
2742 14 : bboxStructArray[i].buffers = bboxStructBuffersPtr[i].data();
2743 :
2744 14 : newSchemaChildren.emplace_back(&bboxStructSchema[i]);
2745 14 : newArrayChildren.emplace_back(&bboxStructArray[i]);
2746 : }
2747 : }
2748 0 : catch (const std::bad_alloc &)
2749 : {
2750 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
2751 : "Out of memory in "
2752 : "OGRArrowWriterLayer::WriteArrowBatchInternal()");
2753 0 : return false;
2754 : }
2755 : }
2756 :
2757 1644 : for (int i = 0; i < static_cast<int>(schema->n_children); ++i)
2758 : {
2759 1518 : if (!cpl::contains(oSetReferencedFieldsInArraySchema, i))
2760 : {
2761 4 : if (m_osFIDColumn.empty() &&
2762 2 : strcmp(schema->children[i]->name, pszFIDName) == 0)
2763 : {
2764 : // If the input data contains a FID column, but the output data
2765 : // does not, then ignore it.
2766 : }
2767 : else
2768 : {
2769 0 : CPLError(CE_Failure, CPLE_AppDefined,
2770 : "Found field '%s' in array schema that does not exist "
2771 : "in layer schema",
2772 0 : schema->children[i]->name);
2773 0 : return false;
2774 : }
2775 : }
2776 : }
2777 :
2778 : // ImportSchema() would release the schema, but we don't want that
2779 : // So copy the structure content into a local variable, and override its
2780 : // release callback to a no-op. This may be a bit fragile, but it doesn't
2781 : // look like ImportSchema implementation tries to access the C ArrowSchema
2782 : // after it has been called.
2783 126 : struct ArrowSchema lSchema = *schema;
2784 126 : schema = &lSchema;
2785 126 : CPL_IGNORE_RET_VAL(schema);
2786 :
2787 126 : lSchema.n_children = newSchemaChildren.size();
2788 126 : lSchema.children = newSchemaChildren.data();
2789 :
2790 126 : lSchema.release = DummyFreeSchema;
2791 252 : auto poSchemaResult = arrow::ImportSchema(&lSchema);
2792 126 : CPLAssert(lSchema.release == nullptr);
2793 126 : if (!poSchemaResult.ok())
2794 : {
2795 0 : CPLError(CE_Failure, CPLE_AppDefined, "ImportSchema() failed with %s",
2796 0 : poSchemaResult.status().message().c_str());
2797 0 : return false;
2798 : }
2799 252 : auto poSchema = *poSchemaResult;
2800 :
2801 : // Hack the array to use the new children we've computed above
2802 : // but make sure the original release() callback sees the original children
2803 : struct ArrayReleaser
2804 : {
2805 : struct ArrowArray ori_array
2806 : {
2807 : };
2808 :
2809 126 : explicit ArrayReleaser(struct ArrowArray *array)
2810 126 : {
2811 126 : memcpy(&ori_array, array, sizeof(*array));
2812 126 : array->release = ArrayReleaser::release;
2813 126 : array->private_data = this;
2814 126 : }
2815 :
2816 126 : static void release(struct ArrowArray *array)
2817 : {
2818 126 : struct ArrayReleaser *releaser =
2819 : static_cast<struct ArrayReleaser *>(array->private_data);
2820 126 : memcpy(array, &(releaser->ori_array), sizeof(*array));
2821 126 : CPLAssert(array->release != nullptr);
2822 126 : array->release(array);
2823 126 : CPLAssert(array->release == nullptr);
2824 126 : delete releaser;
2825 126 : }
2826 : };
2827 :
2828 : // Must be allocated on the heap, since ArrayReleaser::release() will be
2829 : // called after this method has ended.
2830 126 : ArrayReleaser *releaser = new ArrayReleaser(array);
2831 126 : array->private_data = releaser;
2832 126 : array->n_children = newArrayChildren.size();
2833 : // cppcheck-suppress autoVariables
2834 126 : array->children = newArrayChildren.data();
2835 :
2836 : // Process geometry columns:
2837 : // - if the output encoding is WKB, then just note the geometry type and
2838 : // envelope.
2839 : // - otherwise convert to the output encoding.
2840 126 : int nBuilderIdx = 0;
2841 126 : if (!m_osFIDColumn.empty())
2842 : {
2843 2 : nBuilderIdx++;
2844 : }
2845 : std::map<std::string, std::shared_ptr<arrow::Array>>
2846 252 : oMapGeomFieldNameToArray;
2847 252 : for (int i = 0; i < nGeomFieldCount; ++i, ++nBuilderIdx)
2848 : {
2849 : const char *pszThisGeomFieldName =
2850 126 : m_poFeatureDefn->GetGeomFieldDefn(i)->GetNameRef();
2851 126 : int nIdx = poSchema->GetFieldIndex(pszThisGeomFieldName);
2852 126 : if (nIdx < 0)
2853 : {
2854 3 : if (nGeomFieldCount == 1)
2855 3 : nIdx = poSchema->GetFieldIndex(pszSingleGeomFieldName);
2856 3 : if (nIdx < 0)
2857 : {
2858 0 : CPLError(CE_Failure, CPLE_AppDefined,
2859 : "Cannot find geometry field '%s' in schema",
2860 : pszThisGeomFieldName);
2861 0 : return false;
2862 : }
2863 : }
2864 :
2865 126 : if (strcmp(lSchema.children[nIdx]->format, "z") != 0 &&
2866 1 : strcmp(lSchema.children[nIdx]->format, "Z") != 0)
2867 : {
2868 0 : CPLError(CE_Failure, CPLE_AppDefined,
2869 : "Type of geometry field '%s' is not binary, but '%s'",
2870 0 : pszThisGeomFieldName, lSchema.children[nIdx]->format);
2871 0 : return false;
2872 : }
2873 :
2874 126 : const auto psGeomArray = array->children[nIdx];
2875 126 : const uint8_t *pabyValidity =
2876 126 : psGeomArray->null_count != 0
2877 126 : ? static_cast<const uint8_t *>(psGeomArray->buffers[0])
2878 : : nullptr;
2879 126 : const bool bUseOffsets32 =
2880 126 : (strcmp(lSchema.children[nIdx]->format, "z") == 0);
2881 126 : const uint32_t *panOffsets32 =
2882 126 : static_cast<const uint32_t *>(psGeomArray->buffers[1]) +
2883 126 : psGeomArray->offset;
2884 126 : const uint64_t *panOffsets64 =
2885 126 : static_cast<const uint64_t *>(psGeomArray->buffers[1]) +
2886 126 : psGeomArray->offset;
2887 126 : GByte *pabyData =
2888 126 : static_cast<GByte *>(const_cast<void *>(psGeomArray->buffers[2]));
2889 126 : OGREnvelope sEnvelope;
2890 126 : auto poBuilder = m_apoBuilders[nBuilderIdx].get();
2891 :
2892 681 : for (size_t iRow = 0; iRow < static_cast<size_t>(psGeomArray->length);
2893 : ++iRow)
2894 : {
2895 555 : bool bValidGeom = false;
2896 :
2897 1059 : if (!pabyValidity ||
2898 504 : TestBit(pabyValidity,
2899 504 : static_cast<size_t>(iRow + psGeomArray->offset)))
2900 : {
2901 439 : const auto nLen =
2902 439 : bUseOffsets32 ? static_cast<size_t>(panOffsets32[iRow + 1] -
2903 429 : panOffsets32[iRow])
2904 10 : : static_cast<size_t>(panOffsets64[iRow + 1] -
2905 10 : panOffsets64[iRow]);
2906 439 : GByte *pabyWkb =
2907 439 : pabyData + (bUseOffsets32
2908 429 : ? panOffsets32[iRow]
2909 10 : : static_cast<size_t>(panOffsets64[iRow]));
2910 439 : if (m_aeGeomEncoding[i] == OGRArrowGeomEncoding::WKB)
2911 : {
2912 171 : FixupWKBGeometryBeforeWriting(pabyWkb, nLen);
2913 :
2914 171 : uint32_t nType = 0;
2915 171 : bool bNeedSwap = false;
2916 171 : if (OGRWKBGetGeomType(pabyWkb, nLen, bNeedSwap, nType))
2917 : {
2918 171 : m_oSetWrittenGeometryTypes[i].insert(
2919 171 : static_cast<OGRwkbGeometryType>(nType));
2920 171 : if (OGRWKBGetBoundingBox(pabyWkb, nLen, sEnvelope))
2921 : {
2922 171 : bValidGeom = true;
2923 171 : m_aoEnvelopes[i].Merge(sEnvelope);
2924 :
2925 171 : if (m_bWriteBBoxStruct)
2926 : {
2927 43 : aadfMinX[i].push_back(
2928 43 : castToFloatDown(sEnvelope.MinX));
2929 43 : aadfMinY[i].push_back(
2930 43 : castToFloatDown(sEnvelope.MinY));
2931 43 : aadfMaxX[i].push_back(
2932 43 : castToFloatUp(sEnvelope.MaxX));
2933 43 : aadfMaxY[i].push_back(
2934 43 : castToFloatUp(sEnvelope.MaxY));
2935 : }
2936 : }
2937 : }
2938 : }
2939 : else
2940 : {
2941 268 : size_t nBytesConsumedOut = 0;
2942 268 : OGRGeometry *poGeometry = nullptr;
2943 268 : OGRGeometryFactory::createFromWkb(
2944 : pabyWkb, nullptr, &poGeometry, nLen, wkbVariantIso,
2945 : nBytesConsumedOut);
2946 268 : if (BuildGeometry(poGeometry, i, poBuilder) != OGRERR_NONE)
2947 : {
2948 0 : delete poGeometry;
2949 0 : return false;
2950 : }
2951 268 : bValidGeom = true;
2952 268 : if (m_bWriteBBoxStruct)
2953 : {
2954 0 : poGeometry->getEnvelope(&sEnvelope);
2955 0 : aadfMinX[i].push_back(castToFloatDown(sEnvelope.MinX));
2956 0 : aadfMinY[i].push_back(castToFloatDown(sEnvelope.MinY));
2957 0 : aadfMaxX[i].push_back(castToFloatUp(sEnvelope.MaxX));
2958 0 : aadfMaxY[i].push_back(castToFloatUp(sEnvelope.MaxY));
2959 : }
2960 268 : delete poGeometry;
2961 : }
2962 : }
2963 : else
2964 : {
2965 116 : if (m_aeGeomEncoding[i] != OGRArrowGeomEncoding::WKB)
2966 : {
2967 81 : if (BuildGeometry(nullptr, i, poBuilder) != OGRERR_NONE)
2968 0 : return false;
2969 : }
2970 : }
2971 :
2972 555 : if (!bValidGeom && m_bWriteBBoxStruct)
2973 : {
2974 6 : if ((bboxStructSchema[i].flags & ARROW_FLAG_NULLABLE))
2975 : {
2976 6 : bboxStructArray[i].null_count++;
2977 6 : aabyBboxStructValidity[i][iRow / 8] &=
2978 6 : ~(1 << static_cast<int>(iRow % 8));
2979 : }
2980 6 : aadfMinX[i].push_back(0.0f);
2981 6 : aadfMinY[i].push_back(0.0f);
2982 6 : aadfMaxX[i].push_back(0.0f);
2983 6 : aadfMaxY[i].push_back(0.0f);
2984 : }
2985 : }
2986 :
2987 126 : if (m_aeGeomEncoding[i] != OGRArrowGeomEncoding::WKB)
2988 : {
2989 0 : std::shared_ptr<arrow::Array> geomArray;
2990 81 : auto status = poBuilder->Finish(&geomArray);
2991 81 : if (!status.ok())
2992 : {
2993 0 : CPLError(CE_Failure, CPLE_AppDefined,
2994 : "builder::Finish() for field %s failed with %s",
2995 0 : pszThisGeomFieldName, status.message().c_str());
2996 0 : return false;
2997 : }
2998 162 : oMapGeomFieldNameToArray[pszThisGeomFieldName] =
2999 162 : std::move(geomArray);
3000 : }
3001 : }
3002 :
3003 : auto poRecordBatchResult =
3004 252 : arrow::ImportRecordBatch(array, std::move(poSchema));
3005 126 : if (!poRecordBatchResult.ok())
3006 : {
3007 0 : CPLError(CE_Failure, CPLE_AppDefined,
3008 : "ImportRecordBatch() failed with %s",
3009 0 : poRecordBatchResult.status().message().c_str());
3010 0 : return false;
3011 : }
3012 252 : auto poRecordBatch = *poRecordBatchResult;
3013 :
3014 126 : if (!(bRebuildBatch || !oMapGeomFieldNameToArray.empty()))
3015 : {
3016 994 : for (int i = 0; i < m_poSchema->num_fields(); ++i)
3017 : {
3018 : const auto oIter =
3019 952 : oMapGeomFieldNameToArray.find(m_poSchema->field(i)->name());
3020 952 : auto l_array = (oIter != oMapGeomFieldNameToArray.end())
3021 0 : ? oIter->second
3022 1904 : : poRecordBatch->column(i);
3023 1904 : const auto schemaType = m_poSchema->field(i)->type();
3024 1904 : const auto arrayType = l_array->type();
3025 1904 : if (schemaType->id() != arrow::Type::EXTENSION &&
3026 952 : arrayType->id() == arrow::Type::EXTENSION)
3027 : {
3028 0 : bRebuildBatch = true;
3029 : }
3030 952 : else if (schemaType->id() != arrayType->id())
3031 : {
3032 3 : CPLDebug(
3033 : "Arrow",
3034 : "Field idx=%d name='%s', schema type=%s, array type=%s", i,
3035 1 : m_poSchema->field(i)->name().c_str(),
3036 2 : schemaType->ToString().c_str(),
3037 2 : arrayType->ToString().c_str());
3038 : }
3039 : }
3040 : }
3041 :
3042 : // below assertion commented out since it is not strictly necessary, but
3043 : // reflects what ImportRecordBatch() does.
3044 : // CPLAssert(array->release == nullptr);
3045 :
3046 : // We may need to reconstruct a final record batch that perfectly matches
3047 : // the expected schema.
3048 126 : if (bRebuildBatch || !oMapGeomFieldNameToArray.empty())
3049 : {
3050 84 : std::vector<std::shared_ptr<arrow::Array>> apoArrays;
3051 662 : for (int i = 0; i < m_poSchema->num_fields(); ++i)
3052 : {
3053 : const auto oIter =
3054 578 : oMapGeomFieldNameToArray.find(m_poSchema->field(i)->name());
3055 578 : if (oIter != oMapGeomFieldNameToArray.end())
3056 81 : apoArrays.emplace_back(oIter->second);
3057 : else
3058 497 : apoArrays.emplace_back(poRecordBatch->column(i));
3059 :
3060 578 : auto expectedFieldType = m_poSchema->field(i)->type();
3061 578 : if (expectedFieldType->id() == arrow::Type::EXTENSION)
3062 : {
3063 0 : auto extensionType = cpl::down_cast<arrow::ExtensionType *>(
3064 : expectedFieldType.get());
3065 0 : expectedFieldType = extensionType->storage_type();
3066 : }
3067 :
3068 578 : if (apoArrays.back()->type()->id() == arrow::Type::EXTENSION)
3069 : {
3070 0 : apoArrays.back() =
3071 0 : std::static_pointer_cast<arrow::ExtensionArray>(
3072 0 : apoArrays.back())
3073 0 : ->storage();
3074 : }
3075 :
3076 578 : if (apoArrays.back()->type()->id() != expectedFieldType->id())
3077 : {
3078 0 : CPLError(
3079 : CE_Failure, CPLE_AppDefined,
3080 : "Field '%s' of unexpected type. Got '%s', expected '%s'",
3081 0 : m_poSchema->field(i)->name().c_str(),
3082 0 : apoArrays.back()->type()->name().c_str(),
3083 0 : expectedFieldType->name().c_str());
3084 0 : return false;
3085 : }
3086 : }
3087 336 : poRecordBatchResult = arrow::RecordBatch::Make(
3088 252 : m_poSchema, poRecordBatch->num_rows(), std::move(apoArrays));
3089 84 : if (!poRecordBatchResult.ok())
3090 : {
3091 0 : CPLError(CE_Failure, CPLE_AppDefined,
3092 : "RecordBatch::Make() failed with %s",
3093 0 : poRecordBatchResult.status().message().c_str());
3094 0 : return false;
3095 : }
3096 84 : poRecordBatch = *poRecordBatchResult;
3097 : }
3098 :
3099 126 : if (writeBatch(poRecordBatch))
3100 : {
3101 126 : m_nFeatureCount += poRecordBatch->num_rows();
3102 126 : return true;
3103 : }
3104 0 : return false;
3105 : #endif
3106 : }
3107 :
3108 : #endif /* OGARROWWRITERLAYER_HPP_INCLUDED */
|
