Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: Arrow generic code
4 : * Purpose: Arrow generic code
5 : * Author: Even Rouault, <even.rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2022, Planet Labs
9 : *
10 : * SPDX-License-Identifier: MIT
11 : ****************************************************************************/
12 :
13 : #ifndef OGARROWWRITERLAYER_HPP_INCLUDED
14 : #define OGARROWWRITERLAYER_HPP_INCLUDED
15 :
16 : #include "ogr_arrow.h"
17 :
18 : #include "cpl_json.h"
19 : #include "cpl_time.h"
20 :
21 : #include "ogrlayerarrow.h"
22 : #include "ogr_wkb.h"
23 :
24 : #include <array>
25 : #include <cinttypes>
26 : #include <limits>
27 :
28 : static constexpr int TZFLAG_UNINITIALIZED = -1;
29 :
30 : #define OGR_ARROW_RETURN_NOT_OK(status, ret_value) \
31 : do \
32 : { \
33 : if (!(status).ok()) \
34 : { \
35 : CPLError(CE_Failure, CPLE_AppDefined, "%s failed", \
36 : ARROW_STRINGIFY(status)); \
37 : return (ret_value); \
38 : } \
39 : } while (false)
40 :
41 : #define OGR_ARROW_RETURN_FALSE_NOT_OK(status) \
42 : OGR_ARROW_RETURN_NOT_OK(status, false)
43 :
44 : #define OGR_ARROW_RETURN_OGRERR_NOT_OK(status) \
45 : OGR_ARROW_RETURN_NOT_OK(status, OGRERR_FAILURE)
46 :
47 : #define OGR_ARROW_PROPAGATE_OGRERR(ret_value) \
48 : do \
49 : { \
50 : if ((ret_value) != OGRERR_NONE) \
51 : return OGRERR_FAILURE; \
52 : } while (0)
53 :
54 : /************************************************************************/
55 : /* OGRArrowWriterLayer() */
56 : /************************************************************************/
57 :
58 422 : inline OGRArrowWriterLayer::OGRArrowWriterLayer(
59 : arrow::MemoryPool *poMemoryPool,
60 : const std::shared_ptr<arrow::io::OutputStream> &poOutputStream,
61 422 : const char *pszLayerName)
62 422 : : m_poMemoryPool(poMemoryPool), m_poOutputStream(poOutputStream)
63 : {
64 422 : m_poFeatureDefn = new OGRFeatureDefn(pszLayerName);
65 422 : m_poFeatureDefn->SetGeomType(wkbNone);
66 422 : m_poFeatureDefn->Reference();
67 422 : SetDescription(pszLayerName);
68 422 : }
69 :
70 : /************************************************************************/
71 : /* ~OGRArrowWriterLayer() */
72 : /************************************************************************/
73 :
74 422 : inline OGRArrowWriterLayer::~OGRArrowWriterLayer()
75 : {
76 422 : CPLDebug("ARROW", "Memory pool (writer layer): bytes_allocated = %" PRId64,
77 422 : m_poMemoryPool->bytes_allocated());
78 422 : CPLDebug("ARROW", "Memory pool (writer layer): max_memory = %" PRId64,
79 422 : m_poMemoryPool->max_memory());
80 :
81 422 : m_poFeatureDefn->Release();
82 422 : }
83 :
84 : /************************************************************************/
85 : /* FinalizeWriting() */
86 : /************************************************************************/
87 :
88 410 : inline bool OGRArrowWriterLayer::FinalizeWriting()
89 : {
90 410 : bool ret = true;
91 :
92 410 : if (!IsFileWriterCreated())
93 : {
94 281 : CreateWriter();
95 : }
96 410 : if (IsFileWriterCreated())
97 : {
98 410 : PerformStepsBeforeFinalFlushGroup();
99 :
100 410 : if (!m_apoBuilders.empty() && m_apoFieldsFromArrowSchema.empty())
101 235 : ret = FlushGroup();
102 :
103 410 : if (!CloseFileWriter())
104 0 : ret = false;
105 : }
106 :
107 410 : return ret;
108 : }
109 :
110 : /************************************************************************/
111 : /* RemoveIDFromMemberOfEnsembles() */
112 : /************************************************************************/
113 :
114 : /* static */
115 : inline void
116 392 : OGRArrowWriterLayer::RemoveIDFromMemberOfEnsembles(CPLJSONObject &obj)
117 : {
118 : // Remove "id" from members of datum ensembles for compatibility with
119 : // older PROJ versions
120 : // Cf https://github.com/opengeospatial/geoparquet/discussions/110
121 : // and https://github.com/OSGeo/PROJ/pull/3221
122 392 : if (obj.GetType() == CPLJSONObject::Type::Object)
123 : {
124 498 : for (auto &subObj : obj.GetChildren())
125 : {
126 380 : RemoveIDFromMemberOfEnsembles(subObj);
127 : }
128 : }
129 302 : else if (obj.GetType() == CPLJSONObject::Type::Array &&
130 302 : obj.GetName() == "members")
131 : {
132 0 : for (auto &subObj : obj.ToArray())
133 : {
134 0 : subObj.Delete("id");
135 : }
136 : }
137 392 : }
138 :
139 : /************************************************************************/
140 : /* IdentifyCRS() */
141 : /************************************************************************/
142 :
143 : /* static */
144 : inline OGRSpatialReference
145 26 : OGRArrowWriterLayer::IdentifyCRS(const OGRSpatialReference *poSRS)
146 : {
147 26 : OGRSpatialReference oSRSIdentified(*poSRS);
148 :
149 26 : if (poSRS->GetAuthorityName(nullptr) == nullptr)
150 : {
151 : // Try to find a registered CRS that matches the input one
152 4 : int nEntries = 0;
153 4 : int *panConfidence = nullptr;
154 : OGRSpatialReferenceH *pahSRS =
155 4 : poSRS->FindMatches(nullptr, &nEntries, &panConfidence);
156 :
157 : // If there are several matches >= 90%, take the only one
158 : // that is EPSG
159 4 : int iOtherAuthority = -1;
160 4 : int iEPSG = -1;
161 4 : const char *const apszOptions[] = {
162 : "IGNORE_DATA_AXIS_TO_SRS_AXIS_MAPPING=YES", nullptr};
163 4 : int iConfidenceBestMatch = -1;
164 6 : for (int iSRS = 0; iSRS < nEntries; iSRS++)
165 : {
166 4 : auto poCandidateCRS = OGRSpatialReference::FromHandle(pahSRS[iSRS]);
167 4 : if (panConfidence[iSRS] < iConfidenceBestMatch ||
168 4 : panConfidence[iSRS] < 70)
169 : {
170 : break;
171 : }
172 3 : if (poSRS->IsSame(poCandidateCRS, apszOptions))
173 : {
174 : const char *pszAuthName =
175 3 : poCandidateCRS->GetAuthorityName(nullptr);
176 3 : if (pszAuthName != nullptr && EQUAL(pszAuthName, "EPSG"))
177 : {
178 2 : iOtherAuthority = -2;
179 2 : if (iEPSG < 0)
180 : {
181 2 : iConfidenceBestMatch = panConfidence[iSRS];
182 2 : iEPSG = iSRS;
183 : }
184 : else
185 : {
186 0 : iEPSG = -1;
187 0 : break;
188 : }
189 : }
190 1 : else if (iEPSG < 0 && pszAuthName != nullptr)
191 : {
192 1 : if (EQUAL(pszAuthName, "OGC"))
193 : {
194 : const char *pszAuthCode =
195 1 : poCandidateCRS->GetAuthorityCode(nullptr);
196 1 : if (pszAuthCode && EQUAL(pszAuthCode, "CRS84"))
197 : {
198 1 : iOtherAuthority = iSRS;
199 1 : break;
200 : }
201 : }
202 0 : else if (iOtherAuthority == -1)
203 : {
204 0 : iConfidenceBestMatch = panConfidence[iSRS];
205 0 : iOtherAuthority = iSRS;
206 : }
207 : else
208 0 : iOtherAuthority = -2;
209 : }
210 : }
211 : }
212 4 : if (iEPSG >= 0)
213 : {
214 2 : oSRSIdentified = *OGRSpatialReference::FromHandle(pahSRS[iEPSG]);
215 : }
216 2 : else if (iOtherAuthority >= 0)
217 : {
218 : oSRSIdentified =
219 1 : *OGRSpatialReference::FromHandle(pahSRS[iOtherAuthority]);
220 : }
221 4 : OSRFreeSRSArray(pahSRS);
222 4 : CPLFree(panConfidence);
223 : }
224 :
225 26 : return oSRSIdentified;
226 : }
227 :
228 : /************************************************************************/
229 : /* CreateSchemaCommon() */
230 : /************************************************************************/
231 :
232 410 : inline void OGRArrowWriterLayer::CreateSchemaCommon()
233 : {
234 410 : CPLAssert(static_cast<int>(m_aeGeomEncoding.size()) ==
235 : m_poFeatureDefn->GetGeomFieldCount());
236 :
237 820 : std::vector<std::shared_ptr<arrow::Field>> fields;
238 410 : bool bNeedGDALSchema = false;
239 :
240 410 : m_anTZFlag.resize(m_poFeatureDefn->GetFieldCount(), TZFLAG_UNINITIALIZED);
241 :
242 410 : if (!m_osFIDColumn.empty())
243 : {
244 18 : bNeedGDALSchema = true;
245 18 : fields.emplace_back(arrow::field(m_osFIDColumn, arrow::int64(), false));
246 : }
247 :
248 410 : if (!m_apoFieldsFromArrowSchema.empty())
249 : {
250 119 : fields.insert(fields.end(), m_apoFieldsFromArrowSchema.begin(),
251 238 : m_apoFieldsFromArrowSchema.end());
252 : }
253 :
254 1064 : for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i)
255 : {
256 654 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
257 654 : std::shared_ptr<arrow::DataType> dt;
258 654 : const auto eDT = poFieldDefn->GetType();
259 654 : const auto eSubDT = poFieldDefn->GetSubType();
260 654 : const auto &osDomainName = poFieldDefn->GetDomainName();
261 654 : const OGRFieldDomain *poFieldDomain = nullptr;
262 654 : const int nWidth = poFieldDefn->GetWidth();
263 654 : if (!osDomainName.empty())
264 : {
265 4 : const auto oIter = m_oMapFieldDomains.find(osDomainName);
266 4 : if (oIter == m_oMapFieldDomains.end())
267 : {
268 0 : CPLError(CE_Warning, CPLE_AppDefined,
269 : "Field %s references domain %s, but the later one "
270 : "has not been created",
271 : poFieldDefn->GetNameRef(), osDomainName.c_str());
272 : }
273 : else
274 : {
275 4 : poFieldDomain = oIter->second.get();
276 : }
277 : }
278 654 : switch (eDT)
279 : {
280 65 : case OFTInteger:
281 65 : if (eSubDT == OFSTBoolean)
282 4 : dt = arrow::boolean();
283 61 : else if (eSubDT == OFSTInt16)
284 4 : dt = arrow::int16();
285 : else
286 57 : dt = arrow::int32();
287 65 : if (poFieldDomain != nullptr)
288 : {
289 4 : dt = arrow::dictionary(dt, arrow::utf8());
290 : }
291 65 : break;
292 :
293 38 : case OFTInteger64:
294 38 : dt = arrow::int64();
295 38 : if (poFieldDomain != nullptr)
296 : {
297 0 : dt = arrow::dictionary(dt, arrow::utf8());
298 : }
299 38 : break;
300 :
301 54 : case OFTReal:
302 : {
303 54 : const int nPrecision = poFieldDefn->GetPrecision();
304 54 : if (nWidth != 0 && nPrecision != 0)
305 : {
306 : // Since arrow 18.0, we could use arrow::smallest_decimal()
307 : // to return the smallest representation (i.e. possibly
308 : // decimal32 and decimal64). But for now keep decimal128
309 : // as the minimum for backwards compatibility.
310 : // GetValueDecimal() and other functions in
311 : // ogrlayerarrow.cpp would have to be adapted for decimal32
312 : // and decimal64 compatibility.
313 10 : if (nWidth > 38)
314 0 : dt = arrow::decimal256(nWidth, nPrecision);
315 : else
316 10 : dt = arrow::decimal128(nWidth, nPrecision);
317 : }
318 44 : else if (eSubDT == OFSTFloat32)
319 7 : dt = arrow::float32();
320 : else
321 37 : dt = arrow::float64();
322 54 : break;
323 : }
324 :
325 277 : case OFTString:
326 : case OFTWideString:
327 277 : if ((eSubDT != OFSTNone && eSubDT != OFSTJSON) || nWidth > 0)
328 2 : bNeedGDALSchema = true;
329 277 : dt = arrow::utf8();
330 277 : break;
331 :
332 19 : case OFTBinary:
333 19 : if (nWidth != 0)
334 4 : dt = arrow::fixed_size_binary(nWidth);
335 : else
336 15 : dt = arrow::binary();
337 19 : break;
338 :
339 48 : case OFTIntegerList:
340 48 : if (eSubDT == OFSTBoolean)
341 8 : dt = arrow::list(arrow::boolean());
342 40 : else if (eSubDT == OFSTInt16)
343 0 : dt = arrow::list(arrow::int16());
344 : else
345 40 : dt = arrow::list(arrow::int32());
346 48 : break;
347 :
348 20 : case OFTInteger64List:
349 20 : dt = arrow::list(arrow::int64());
350 20 : break;
351 :
352 35 : case OFTRealList:
353 35 : if (eSubDT == OFSTFloat32)
354 11 : dt = arrow::list(arrow::float32());
355 : else
356 24 : dt = arrow::list(arrow::float64());
357 35 : break;
358 :
359 12 : case OFTStringList:
360 : case OFTWideStringList:
361 12 : dt = arrow::list(arrow::utf8());
362 12 : break;
363 :
364 31 : case OFTDate:
365 31 : dt = arrow::date32();
366 31 : break;
367 :
368 8 : case OFTTime:
369 8 : dt = arrow::time32(arrow::TimeUnit::MILLI);
370 8 : break;
371 :
372 47 : case OFTDateTime:
373 : {
374 47 : const int nTZFlag = poFieldDefn->GetTZFlag();
375 47 : if (nTZFlag >= OGR_TZFLAG_MIXED_TZ)
376 : {
377 12 : m_anTZFlag[i] = nTZFlag;
378 : }
379 47 : dt = arrow::timestamp(arrow::TimeUnit::MILLI);
380 47 : break;
381 : }
382 : }
383 :
384 654 : auto field = arrow::field(poFieldDefn->GetNameRef(), std::move(dt),
385 1962 : poFieldDefn->IsNullable());
386 654 : if (eDT == OFTString && eSubDT == OFSTJSON)
387 : {
388 83 : auto kvMetadata = std::make_shared<arrow::KeyValueMetadata>();
389 83 : kvMetadata->Append(ARROW_EXTENSION_NAME_KEY,
390 : EXTENSION_NAME_ARROW_JSON);
391 83 : field = field->WithMetadata(kvMetadata);
392 : }
393 :
394 654 : fields.emplace_back(std::move(field));
395 654 : if (poFieldDefn->GetAlternativeNameRef()[0])
396 2 : bNeedGDALSchema = true;
397 654 : if (!poFieldDefn->GetComment().empty())
398 3 : bNeedGDALSchema = true;
399 : }
400 :
401 814 : for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
402 : {
403 404 : const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
404 404 : const auto eGType = poGeomFieldDefn->GetType();
405 : const int nDim =
406 404 : 2 + (OGR_GT_HasZ(eGType) ? 1 : 0) + (OGR_GT_HasM(eGType) ? 1 : 0);
407 :
408 404 : const bool pointFieldNullable = GetDriverUCName() == "PARQUET";
409 :
410 : // Fixed Size List GeoArrow encoding
411 : const auto getFixedSizeListOfPoint =
412 228 : [nDim, eGType, pointFieldNullable]()
413 : {
414 : return arrow::fixed_size_list(
415 146 : arrow::field(nDim == 2 ? "xy"
416 30 : : nDim == 3 ? (OGR_GT_HasZ(eGType) ? "xyz" : "xym")
417 : : "xyzm",
418 : arrow::float64(), pointFieldNullable),
419 116 : nDim);
420 404 : };
421 :
422 : // Struct GeoArrow encoding
423 1212 : auto xField(arrow::field("x", arrow::float64(), false));
424 1212 : auto yField(arrow::field("y", arrow::float64(), false));
425 : std::vector<std::shared_ptr<arrow::Field>> pointFields{
426 : arrow::field("x", arrow::float64(), false),
427 2424 : arrow::field("y", arrow::float64(), false)};
428 404 : if (OGR_GT_HasZ(eGType))
429 : pointFields.emplace_back(
430 121 : arrow::field("z", arrow::float64(), false));
431 404 : if (OGR_GT_HasM(eGType))
432 : pointFields.emplace_back(
433 52 : arrow::field("m", arrow::float64(), false));
434 808 : auto pointStructType(arrow::struct_(std::move(pointFields)));
435 :
436 40 : const auto getListOfVertices = [&getFixedSizeListOfPoint]()
437 : {
438 80 : return arrow::list(std::make_shared<arrow::Field>(
439 120 : "vertices", getFixedSizeListOfPoint()));
440 404 : };
441 :
442 22 : const auto getListOfRings = [&getListOfVertices]()
443 : {
444 : return arrow::list(
445 44 : std::make_shared<arrow::Field>("rings", getListOfVertices()));
446 404 : };
447 :
448 104 : const auto getListOfVerticesStruct = [&pointStructType]()
449 : {
450 : return arrow::list(
451 208 : std::make_shared<arrow::Field>("vertices", pointStructType));
452 404 : };
453 :
454 60 : const auto getListOfRingsStruct = [&getListOfVerticesStruct]()
455 : {
456 120 : return arrow::list(std::make_shared<arrow::Field>(
457 180 : "rings", getListOfVerticesStruct()));
458 404 : };
459 :
460 404 : std::shared_ptr<arrow::DataType> dt;
461 404 : switch (m_aeGeomEncoding[i])
462 : {
463 136 : case OGRArrowGeomEncoding::WKB:
464 : #if ARROW_VERSION_MAJOR >= 21
465 : if (m_bUseArrowWKBExtension)
466 : {
467 : CPLJSONDocument oMetadataDoc;
468 :
469 : const auto poSRS = poGeomFieldDefn->GetSpatialRef();
470 : if (poSRS)
471 : {
472 : OGRSpatialReference oSRSIdentified(IdentifyCRS(poSRS));
473 :
474 : // CRS encoded as PROJJSON
475 : char *pszPROJJSON = nullptr;
476 : oSRSIdentified.exportToPROJJSON(&pszPROJJSON, nullptr);
477 : CPLJSONDocument oCRSDoc;
478 : CPL_IGNORE_RET_VAL(oCRSDoc.LoadMemory(pszPROJJSON));
479 : CPLFree(pszPROJJSON);
480 : CPLJSONObject oCRSRoot = oCRSDoc.GetRoot();
481 : RemoveIDFromMemberOfEnsembles(oCRSRoot);
482 :
483 : oMetadataDoc.GetRoot().Add("crs", oCRSRoot);
484 : }
485 :
486 : if (m_bEdgesSpherical)
487 : {
488 : oMetadataDoc.GetRoot().Add("edges", "spherical");
489 : }
490 :
491 : const std::string metadata = oMetadataDoc.GetRoot().Format(
492 : CPLJSONObject::PrettyFormat::Plain);
493 : dt = std::make_shared<OGRGeoArrowWkbExtensionType>(
494 : arrow::binary(), metadata);
495 : }
496 : else
497 : #endif
498 : {
499 136 : dt = arrow::binary();
500 : }
501 136 : break;
502 :
503 53 : case OGRArrowGeomEncoding::WKT:
504 53 : dt = arrow::utf8();
505 53 : break;
506 :
507 0 : case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
508 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
509 0 : CPLAssert(false);
510 : break;
511 :
512 9 : case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
513 9 : dt = getFixedSizeListOfPoint();
514 9 : break;
515 :
516 9 : case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
517 9 : dt = getListOfVertices();
518 9 : break;
519 :
520 11 : case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
521 11 : dt = getListOfRings();
522 11 : break;
523 :
524 9 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
525 18 : dt = arrow::list(std::make_shared<arrow::Field>(
526 27 : "points", getFixedSizeListOfPoint()));
527 9 : break;
528 :
529 9 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
530 18 : dt = arrow::list(std::make_shared<arrow::Field>(
531 27 : "linestrings", getListOfVertices()));
532 9 : break;
533 :
534 11 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
535 22 : dt = arrow::list(std::make_shared<arrow::Field>(
536 33 : "polygons", getListOfRings()));
537 11 : break;
538 :
539 31 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
540 31 : dt = pointStructType;
541 31 : break;
542 :
543 22 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
544 22 : dt = getListOfVerticesStruct();
545 22 : break;
546 :
547 30 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
548 30 : dt = getListOfRingsStruct();
549 30 : break;
550 :
551 22 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
552 44 : dt = arrow::list(
553 66 : std::make_shared<arrow::Field>("points", pointStructType));
554 22 : break;
555 :
556 22 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
557 44 : dt = arrow::list(std::make_shared<arrow::Field>(
558 66 : "linestrings", getListOfVerticesStruct()));
559 22 : break;
560 :
561 30 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
562 60 : dt = arrow::list(std::make_shared<arrow::Field>(
563 90 : "polygons", getListOfRingsStruct()));
564 30 : break;
565 : }
566 :
567 : std::shared_ptr<arrow::Field> field(
568 404 : arrow::field(poGeomFieldDefn->GetNameRef(), std::move(dt),
569 1212 : poGeomFieldDefn->IsNullable()));
570 404 : if (m_bWriteFieldArrowExtensionName)
571 : {
572 136 : auto kvMetadata = field->metadata()
573 136 : ? field->metadata()->Copy()
574 136 : : std::make_shared<arrow::KeyValueMetadata>();
575 272 : kvMetadata->Append(
576 : ARROW_EXTENSION_NAME_KEY,
577 136 : GetGeomEncodingAsString(m_aeGeomEncoding[i], false));
578 136 : field = field->WithMetadata(kvMetadata);
579 : }
580 :
581 404 : m_apoBaseStructGeomType.emplace_back(std::move(pointStructType));
582 :
583 404 : fields.emplace_back(std::move(field));
584 : }
585 :
586 410 : if (m_bWriteBBoxStruct)
587 : {
588 398 : for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
589 : {
590 199 : const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
591 597 : auto bbox_field_xmin(arrow::field("xmin", arrow::float32(), false));
592 597 : auto bbox_field_ymin(arrow::field("ymin", arrow::float32(), false));
593 597 : auto bbox_field_xmax(arrow::field("xmax", arrow::float32(), false));
594 597 : auto bbox_field_ymax(arrow::field("ymax", arrow::float32(), false));
595 : auto bbox_field(arrow::field(
596 : CPLGetConfigOption("OGR_PARQUET_COVERING_BBOX_NAME",
597 398 : std::string(poGeomFieldDefn->GetNameRef())
598 199 : .append("_bbox")
599 : .c_str()),
600 1194 : arrow::struct_(
601 199 : {std::move(bbox_field_xmin), std::move(bbox_field_ymin),
602 1194 : std::move(bbox_field_xmax), std::move(bbox_field_ymax)}),
603 1194 : poGeomFieldDefn->IsNullable()));
604 199 : fields.emplace_back(bbox_field);
605 199 : m_apoFieldsBBOX.emplace_back(bbox_field);
606 : }
607 : }
608 :
609 410 : m_aoEnvelopes.resize(m_poFeatureDefn->GetGeomFieldCount());
610 410 : m_oSetWrittenGeometryTypes.resize(m_poFeatureDefn->GetGeomFieldCount());
611 :
612 410 : m_poSchema = arrow::schema(std::move(fields));
613 410 : CPLAssert(m_poSchema);
614 433 : if (bNeedGDALSchema &&
615 23 : CPLTestBool(CPLGetConfigOption(
616 433 : ("OGR_" + GetDriverUCName() + "_WRITE_GDAL_SCHEMA").c_str(),
617 : "YES")))
618 : {
619 46 : CPLJSONObject oRoot;
620 46 : CPLJSONObject oColumns;
621 :
622 23 : if (!m_osFIDColumn.empty())
623 18 : oRoot.Add("fid", m_osFIDColumn);
624 :
625 23 : oRoot.Add("columns", oColumns);
626 210 : for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i)
627 : {
628 187 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
629 374 : CPLJSONObject oColumn;
630 187 : oColumns.Add(poFieldDefn->GetNameRef(), oColumn);
631 187 : oColumn.Add("type", OGR_GetFieldTypeName(poFieldDefn->GetType()));
632 187 : const auto eSubDT = poFieldDefn->GetSubType();
633 187 : if (eSubDT != OFSTNone)
634 56 : oColumn.Add("subtype", OGR_GetFieldSubTypeName(eSubDT));
635 187 : const int nWidth = poFieldDefn->GetWidth();
636 187 : if (nWidth > 0)
637 12 : oColumn.Add("width", nWidth);
638 187 : const int nPrecision = poFieldDefn->GetPrecision();
639 187 : if (nPrecision > 0)
640 6 : oColumn.Add("precision", nPrecision);
641 187 : if (poFieldDefn->GetAlternativeNameRef()[0])
642 2 : oColumn.Add("alternative_name",
643 : poFieldDefn->GetAlternativeNameRef());
644 187 : if (!poFieldDefn->GetComment().empty())
645 3 : oColumn.Add("comment", poFieldDefn->GetComment());
646 : }
647 :
648 23 : auto kvMetadata = m_poSchema->metadata()
649 0 : ? m_poSchema->metadata()->Copy()
650 46 : : std::make_shared<arrow::KeyValueMetadata>();
651 46 : kvMetadata->Append("gdal:schema",
652 46 : oRoot.Format(CPLJSONObject::PrettyFormat::Plain));
653 23 : m_poSchema = m_poSchema->WithMetadata(kvMetadata);
654 23 : CPLAssert(m_poSchema);
655 : }
656 410 : }
657 :
658 : /************************************************************************/
659 : /* FinalizeSchema() */
660 : /************************************************************************/
661 :
662 355 : inline void OGRArrowWriterLayer::FinalizeSchema()
663 : {
664 : // Final tuning of schema taking into actual timezone values
665 : // from features
666 355 : int nArrowIdxFirstField = !m_osFIDColumn.empty() ? 1 : 0;
667 1007 : for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i)
668 : {
669 652 : if (m_anTZFlag[i] >= OGR_TZFLAG_MIXED_TZ)
670 : {
671 12 : const int nOffset = m_anTZFlag[i] == OGR_TZFLAG_UTC
672 12 : ? 0
673 8 : : (m_anTZFlag[i] - OGR_TZFLAG_UTC) * 15;
674 12 : int nHours = static_cast<int>(nOffset / 60); // Round towards zero.
675 12 : const int nMinutes = std::abs(nOffset - nHours * 60);
676 :
677 : const std::string osTZ =
678 : CPLSPrintf("%c%02d:%02d", nOffset >= 0 ? '+' : '-',
679 24 : std::abs(nHours), nMinutes);
680 24 : auto dt = arrow::timestamp(arrow::TimeUnit::MILLI, osTZ);
681 12 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
682 12 : auto field = arrow::field(poFieldDefn->GetNameRef(), std::move(dt),
683 36 : poFieldDefn->IsNullable());
684 24 : auto result = m_poSchema->SetField(nArrowIdxFirstField + i, field);
685 12 : if (!result.ok())
686 : {
687 0 : CPLError(CE_Warning, CPLE_AppDefined,
688 : "Schema::SetField() failed with %s",
689 0 : result.status().message().c_str());
690 : }
691 : else
692 : {
693 12 : m_poSchema = *result;
694 : }
695 : }
696 : }
697 355 : }
698 :
699 : /************************************************************************/
700 : /* AddFieldDomain() */
701 : /************************************************************************/
702 :
703 : inline bool
704 11 : OGRArrowWriterLayer::AddFieldDomain(std::unique_ptr<OGRFieldDomain> &&domain,
705 : std::string &failureReason)
706 : {
707 11 : if (domain->GetDomainType() != OFDT_CODED)
708 : {
709 0 : failureReason = "Only coded field domains are supported by Arrow";
710 0 : return false;
711 : }
712 :
713 : const OGRCodedFieldDomain *poDomain =
714 11 : static_cast<const OGRCodedFieldDomain *>(domain.get());
715 11 : const OGRCodedValue *psIter = poDomain->GetEnumeration();
716 :
717 : auto poStringBuilder =
718 22 : std::make_shared<arrow::StringBuilder>(m_poMemoryPool);
719 :
720 11 : int nLastCode = -1;
721 44 : for (; psIter->pszCode; ++psIter)
722 : {
723 33 : if (CPLGetValueType(psIter->pszCode) != CPL_VALUE_INTEGER)
724 : {
725 0 : failureReason = "Non integer code in domain ";
726 0 : failureReason += domain->GetName();
727 0 : return false;
728 : }
729 33 : int nCode = atoi(psIter->pszCode);
730 33 : if (nCode <= nLastCode || nCode - nLastCode > 100)
731 : {
732 0 : failureReason = "Too sparse codes in domain ";
733 0 : failureReason += domain->GetName();
734 0 : return false;
735 : }
736 33 : for (int i = nLastCode + 1; i < nCode; ++i)
737 : {
738 0 : OGR_ARROW_RETURN_FALSE_NOT_OK(poStringBuilder->AppendNull());
739 : }
740 33 : if (psIter->pszValue)
741 33 : OGR_ARROW_RETURN_FALSE_NOT_OK(
742 : poStringBuilder->Append(psIter->pszValue));
743 : else
744 0 : OGR_ARROW_RETURN_FALSE_NOT_OK(poStringBuilder->AppendNull());
745 33 : nLastCode = nCode;
746 : }
747 :
748 11 : std::shared_ptr<arrow::Array> stringArray;
749 22 : auto status = poStringBuilder->Finish(&stringArray);
750 11 : if (!status.ok())
751 : {
752 0 : CPLError(CE_Failure, CPLE_AppDefined,
753 : "StringArray::Finish() failed with %s",
754 0 : status.message().c_str());
755 0 : return false;
756 : }
757 :
758 11 : m_oMapFieldDomainToStringArray[domain->GetName()] = std::move(stringArray);
759 11 : m_oMapFieldDomains[domain->GetName()] = std::move(domain);
760 11 : return true;
761 : }
762 :
763 : /************************************************************************/
764 : /* GetFieldDomainNames() */
765 : /************************************************************************/
766 :
767 0 : inline std::vector<std::string> OGRArrowWriterLayer::GetFieldDomainNames() const
768 : {
769 0 : std::vector<std::string> names;
770 0 : names.reserve(m_oMapFieldDomains.size());
771 0 : for (const auto &it : m_oMapFieldDomains)
772 : {
773 0 : names.emplace_back(it.first);
774 : }
775 0 : return names;
776 : }
777 :
778 : /************************************************************************/
779 : /* GetFieldDomain() */
780 : /************************************************************************/
781 :
782 : inline const OGRFieldDomain *
783 15 : OGRArrowWriterLayer::GetFieldDomain(const std::string &name) const
784 : {
785 15 : const auto iter = m_oMapFieldDomains.find(name);
786 15 : if (iter == m_oMapFieldDomains.end())
787 11 : return nullptr;
788 4 : return iter->second.get();
789 : }
790 :
791 : /************************************************************************/
792 : /* CreateField() */
793 : /************************************************************************/
794 :
795 655 : inline OGRErr OGRArrowWriterLayer::CreateField(const OGRFieldDefn *poField,
796 : int /* bApproxOK */)
797 : {
798 655 : if (m_poSchema)
799 : {
800 1 : CPLError(CE_Failure, CPLE_NotSupported,
801 : "Cannot add field after a first feature has been written");
802 1 : return OGRERR_FAILURE;
803 : }
804 654 : if (!m_apoFieldsFromArrowSchema.empty())
805 : {
806 0 : CPLError(CE_Failure, CPLE_NotSupported,
807 : "Cannot mix calls to CreateField() and "
808 : "CreateFieldFromArrowSchema()");
809 0 : return OGRERR_FAILURE;
810 : }
811 654 : m_poFeatureDefn->AddFieldDefn(poField);
812 654 : return OGRERR_NONE;
813 : }
814 :
815 : /************************************************************************/
816 : /* OGRLayer::CreateFieldFromArrowSchema() */
817 : /************************************************************************/
818 :
819 998 : inline bool OGRArrowWriterLayer::CreateFieldFromArrowSchema(
820 : const struct ArrowSchema *schema, CSLConstList /*papszOptions*/)
821 : {
822 998 : if (m_poSchema)
823 : {
824 0 : CPLError(CE_Failure, CPLE_NotSupported,
825 : "Cannot add field after a first feature has been written");
826 0 : return false;
827 : }
828 :
829 998 : if (m_poFeatureDefn->GetFieldCount())
830 : {
831 0 : CPLError(CE_Failure, CPLE_NotSupported,
832 : "Cannot mix calls to CreateField() and "
833 : "CreateFieldFromArrowSchema()");
834 0 : return false;
835 : }
836 :
837 998 : if (m_osFIDColumn == schema->name)
838 : {
839 0 : CPLError(CE_Failure, CPLE_AppDefined,
840 : "FID column has the same name as this field: %s",
841 0 : schema->name);
842 0 : return false;
843 : }
844 :
845 35951 : for (auto &apoField : m_apoFieldsFromArrowSchema)
846 : {
847 34953 : if (apoField->name() == schema->name)
848 : {
849 0 : CPLError(CE_Failure, CPLE_AppDefined,
850 0 : "Field of name %s already exists", schema->name);
851 0 : return false;
852 : }
853 : }
854 :
855 998 : if (m_poFeatureDefn->GetGeomFieldIndex(schema->name) >= 0)
856 : {
857 0 : CPLError(CE_Failure, CPLE_AppDefined,
858 0 : "Geometry field of name %s already exists", schema->name);
859 0 : return false;
860 : }
861 :
862 : // ImportField() would release the schema, but we don't want that
863 : // So copy the structure content into a local variable, and override its
864 : // release callback to a no-op. This may be a bit fragile, but it doesn't
865 : // look like ImportField implementation tries to access the C ArrowSchema
866 : // after it has been called.
867 998 : struct ArrowSchema lSchema = *schema;
868 998 : const auto DummyFreeSchema = [](struct ArrowSchema *ptrSchema)
869 998 : { ptrSchema->release = nullptr; };
870 998 : lSchema.release = DummyFreeSchema;
871 1996 : auto result = arrow::ImportField(&lSchema);
872 998 : CPLAssert(lSchema.release == nullptr);
873 998 : if (!result.ok())
874 : {
875 0 : CPLError(CE_Failure, CPLE_AppDefined,
876 : "CreateFieldFromArrowSchema() failed");
877 0 : return false;
878 : }
879 998 : m_apoFieldsFromArrowSchema.emplace_back(std::move(*result));
880 998 : return true;
881 : }
882 :
883 : /************************************************************************/
884 : /* GetPreciseArrowGeomEncoding() */
885 : /************************************************************************/
886 :
887 217 : inline OGRArrowGeomEncoding OGRArrowWriterLayer::GetPreciseArrowGeomEncoding(
888 : OGRArrowGeomEncoding eEncodingType, OGRwkbGeometryType eGType)
889 : {
890 217 : CPLAssert(eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC ||
891 : eEncodingType == OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC);
892 217 : const auto eFlatType = wkbFlatten(eGType);
893 217 : if (eFlatType == wkbPoint)
894 : {
895 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
896 40 : ? OGRArrowGeomEncoding::GEOARROW_FSL_POINT
897 40 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT;
898 : }
899 177 : else if (eFlatType == wkbLineString)
900 : {
901 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
902 31 : ? OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING
903 31 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING;
904 : }
905 146 : else if (eFlatType == wkbPolygon)
906 : {
907 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
908 41 : ? OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON
909 41 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON;
910 : }
911 105 : else if (eFlatType == wkbMultiPoint)
912 : {
913 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
914 31 : ? OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT
915 31 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT;
916 : }
917 74 : else if (eFlatType == wkbMultiLineString)
918 : {
919 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
920 31 : ? OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING
921 31 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING;
922 : }
923 43 : else if (eFlatType == wkbMultiPolygon)
924 : {
925 : return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
926 41 : ? OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON
927 41 : : OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON;
928 : }
929 : else
930 : {
931 2 : CPLError(CE_Failure, CPLE_NotSupported,
932 : "GeoArrow encoding is currently not supported for %s",
933 : OGRGeometryTypeToName(eGType));
934 2 : return eEncodingType;
935 : }
936 : }
937 :
938 : /************************************************************************/
939 : /* GetGeomEncodingAsString() */
940 : /************************************************************************/
941 :
942 : inline const char *
943 652 : OGRArrowWriterLayer::GetGeomEncodingAsString(OGRArrowGeomEncoding eGeomEncoding,
944 : bool bForParquetGeo)
945 : {
946 652 : switch (eGeomEncoding)
947 : {
948 183 : case OGRArrowGeomEncoding::WKB:
949 183 : return bForParquetGeo ? "WKB" : "geoarrow.wkb";
950 111 : case OGRArrowGeomEncoding::WKT:
951 111 : return bForParquetGeo ? "WKT" : "geoarrow.wkt";
952 0 : case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
953 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
954 0 : CPLAssert(false);
955 : break;
956 19 : case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
957 19 : return "geoarrow.point";
958 19 : case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
959 19 : return "geoarrow.linestring";
960 21 : case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
961 21 : return "geoarrow.polygon";
962 19 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
963 19 : return "geoarrow.multipoint";
964 19 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
965 19 : return "geoarrow.multilinestring";
966 21 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
967 21 : return "geoarrow.multipolygon";
968 54 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
969 54 : return bForParquetGeo ? "point" : "geoarrow.point";
970 34 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
971 34 : return bForParquetGeo ? "linestring" : "geoarrow.linestring";
972 42 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
973 42 : return bForParquetGeo ? "polygon" : "geoarrow.polygon";
974 34 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
975 34 : return bForParquetGeo ? "multipoint" : "geoarrow.multipoint";
976 34 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
977 34 : return bForParquetGeo ? "multilinestring"
978 34 : : "geoarrow.multilinestring";
979 42 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
980 42 : return bForParquetGeo ? "multipolygon" : "geoarrow.multipolygon";
981 : }
982 0 : return nullptr;
983 : }
984 :
985 : /************************************************************************/
986 : /* CreateGeomField() */
987 : /************************************************************************/
988 :
989 : inline OGRErr
990 27 : OGRArrowWriterLayer::CreateGeomField(const OGRGeomFieldDefn *poField,
991 : int /* bApproxOK */)
992 : {
993 27 : if (m_poSchema)
994 : {
995 1 : CPLError(CE_Failure, CPLE_NotSupported,
996 : "Cannot add field after a first feature has been written");
997 1 : return OGRERR_FAILURE;
998 : }
999 26 : const auto eGType = poField->GetType();
1000 26 : if (!IsSupportedGeometryType(eGType))
1001 : {
1002 0 : return OGRERR_FAILURE;
1003 : }
1004 :
1005 26 : if (IsSRSRequired() && poField->GetSpatialRef() == nullptr)
1006 : {
1007 0 : CPLError(CE_Warning, CPLE_AppDefined,
1008 : "Geometry column should have an associated CRS");
1009 : }
1010 26 : auto eGeomEncoding = m_eGeomEncoding;
1011 26 : if (eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC ||
1012 26 : eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC)
1013 : {
1014 0 : const auto eEncodingType = eGeomEncoding;
1015 0 : eGeomEncoding = GetPreciseArrowGeomEncoding(eEncodingType, eGType);
1016 0 : if (eGeomEncoding == eEncodingType)
1017 0 : return OGRERR_FAILURE;
1018 : }
1019 26 : m_aeGeomEncoding.push_back(eGeomEncoding);
1020 26 : m_poFeatureDefn->AddGeomFieldDefn(poField);
1021 26 : return OGRERR_NONE;
1022 : }
1023 :
1024 : /************************************************************************/
1025 : /* MakeGeoArrowBuilder() */
1026 : /************************************************************************/
1027 :
1028 : static std::shared_ptr<arrow::ArrayBuilder>
1029 134 : MakeGeoArrowBuilder(arrow::MemoryPool *poMemoryPool, int nDim, int nDepth)
1030 : {
1031 134 : if (nDepth == 0)
1032 104 : return std::make_shared<arrow::FixedSizeListBuilder>(
1033 104 : poMemoryPool, std::make_shared<arrow::DoubleBuilder>(poMemoryPool),
1034 52 : nDim);
1035 : else
1036 164 : return std::make_shared<arrow::ListBuilder>(
1037 246 : poMemoryPool, MakeGeoArrowBuilder(poMemoryPool, nDim, nDepth - 1));
1038 : }
1039 :
1040 : /************************************************************************/
1041 : /* MakeGeoArrowStructBuilder() */
1042 : /************************************************************************/
1043 :
1044 : static std::shared_ptr<arrow::ArrayBuilder>
1045 384 : MakeGeoArrowStructBuilder(arrow::MemoryPool *poMemoryPool, int nDim, int nDepth,
1046 : const std::shared_ptr<arrow::DataType> &eBaseType)
1047 : {
1048 384 : if (nDepth == 0)
1049 : {
1050 155 : std::vector<std::shared_ptr<arrow::ArrayBuilder>> builders;
1051 537 : for (int i = 0; i < nDim; ++i)
1052 : builders.emplace_back(
1053 382 : std::make_shared<arrow::DoubleBuilder>(poMemoryPool));
1054 310 : return std::make_shared<arrow::StructBuilder>(eBaseType, poMemoryPool,
1055 310 : std::move(builders));
1056 : }
1057 : else
1058 458 : return std::make_shared<arrow::ListBuilder>(
1059 458 : poMemoryPool, MakeGeoArrowStructBuilder(poMemoryPool, nDim,
1060 229 : nDepth - 1, eBaseType));
1061 : }
1062 :
1063 : /************************************************************************/
1064 : /* ClearArrayBuilers() */
1065 : /************************************************************************/
1066 :
1067 273 : inline void OGRArrowWriterLayer::ClearArrayBuilers()
1068 : {
1069 273 : m_apoBuilders.clear();
1070 273 : m_apoBuildersBBOXStruct.clear();
1071 273 : m_apoBuildersBBOXXMin.clear();
1072 273 : m_apoBuildersBBOXYMin.clear();
1073 273 : m_apoBuildersBBOXXMax.clear();
1074 273 : m_apoBuildersBBOXYMax.clear();
1075 273 : }
1076 :
1077 : /************************************************************************/
1078 : /* CreateArrayBuilders() */
1079 : /************************************************************************/
1080 :
1081 392 : inline void OGRArrowWriterLayer::CreateArrayBuilders()
1082 : {
1083 392 : m_apoBuilders.reserve(1 + m_poFeatureDefn->GetFieldCount() +
1084 392 : m_poFeatureDefn->GetGeomFieldCount());
1085 :
1086 392 : int nArrowIdx = 0;
1087 392 : if (!m_osFIDColumn.empty())
1088 : {
1089 49 : m_apoBuilders.emplace_back(std::make_shared<arrow::Int64Builder>());
1090 49 : nArrowIdx++;
1091 : }
1092 :
1093 1760 : for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i, ++nArrowIdx)
1094 : {
1095 1368 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
1096 1368 : std::shared_ptr<arrow::ArrayBuilder> builder;
1097 1368 : const auto eSubDT = poFieldDefn->GetSubType();
1098 1368 : switch (poFieldDefn->GetType())
1099 : {
1100 145 : case OFTInteger:
1101 145 : if (eSubDT == OFSTBoolean)
1102 : builder =
1103 12 : std::make_shared<arrow::BooleanBuilder>(m_poMemoryPool);
1104 133 : else if (eSubDT == OFSTInt16)
1105 : builder =
1106 12 : std::make_shared<arrow::Int16Builder>(m_poMemoryPool);
1107 : else
1108 : builder =
1109 121 : std::make_shared<arrow::Int32Builder>(m_poMemoryPool);
1110 145 : break;
1111 :
1112 86 : case OFTInteger64:
1113 86 : builder = std::make_shared<arrow::Int64Builder>(m_poMemoryPool);
1114 86 : break;
1115 :
1116 108 : case OFTReal:
1117 : {
1118 216 : const auto arrowType = m_poSchema->fields()[nArrowIdx]->type();
1119 108 : if (arrowType->id() == arrow::Type::DECIMAL128)
1120 26 : builder = std::make_shared<arrow::Decimal128Builder>(
1121 26 : arrowType, m_poMemoryPool);
1122 82 : else if (arrowType->id() == arrow::Type::DECIMAL256)
1123 0 : builder = std::make_shared<arrow::Decimal256Builder>(
1124 0 : arrowType, m_poMemoryPool);
1125 82 : else if (eSubDT == OFSTFloat32)
1126 : builder =
1127 21 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool);
1128 : else
1129 : builder =
1130 61 : std::make_shared<arrow::DoubleBuilder>(m_poMemoryPool);
1131 108 : break;
1132 : }
1133 :
1134 475 : case OFTString:
1135 : case OFTWideString:
1136 : builder =
1137 475 : std::make_shared<arrow::StringBuilder>(m_poMemoryPool);
1138 475 : break;
1139 :
1140 43 : case OFTBinary:
1141 43 : if (poFieldDefn->GetWidth() != 0)
1142 24 : builder = std::make_shared<arrow::FixedSizeBinaryBuilder>(
1143 24 : arrow::fixed_size_binary(poFieldDefn->GetWidth()),
1144 24 : m_poMemoryPool);
1145 : else
1146 : builder =
1147 31 : std::make_shared<arrow::BinaryBuilder>(m_poMemoryPool);
1148 43 : break;
1149 :
1150 144 : case OFTIntegerList:
1151 : {
1152 144 : std::shared_ptr<arrow::ArrayBuilder> poBaseBuilder;
1153 144 : if (eSubDT == OFSTBoolean)
1154 : poBaseBuilder =
1155 24 : std::make_shared<arrow::BooleanBuilder>(m_poMemoryPool);
1156 120 : else if (eSubDT == OFSTInt16)
1157 : poBaseBuilder =
1158 0 : std::make_shared<arrow::Int16Builder>(m_poMemoryPool);
1159 : else
1160 : poBaseBuilder =
1161 120 : std::make_shared<arrow::Int32Builder>(m_poMemoryPool);
1162 288 : builder = std::make_shared<arrow::ListBuilder>(m_poMemoryPool,
1163 144 : poBaseBuilder);
1164 144 : break;
1165 : }
1166 :
1167 60 : case OFTInteger64List:
1168 60 : builder = std::make_shared<arrow::ListBuilder>(
1169 60 : m_poMemoryPool,
1170 180 : std::make_shared<arrow::Int64Builder>(m_poMemoryPool));
1171 :
1172 60 : break;
1173 :
1174 105 : case OFTRealList:
1175 105 : if (eSubDT == OFSTFloat32)
1176 33 : builder = std::make_shared<arrow::ListBuilder>(
1177 33 : m_poMemoryPool,
1178 99 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1179 : else
1180 72 : builder = std::make_shared<arrow::ListBuilder>(
1181 72 : m_poMemoryPool,
1182 216 : std::make_shared<arrow::DoubleBuilder>(m_poMemoryPool));
1183 105 : break;
1184 :
1185 36 : case OFTStringList:
1186 : case OFTWideStringList:
1187 36 : builder = std::make_shared<arrow::ListBuilder>(
1188 36 : m_poMemoryPool,
1189 108 : std::make_shared<arrow::StringBuilder>(m_poMemoryPool));
1190 :
1191 36 : break;
1192 :
1193 47 : case OFTDate:
1194 : builder =
1195 47 : std::make_shared<arrow::Date32Builder>(m_poMemoryPool);
1196 47 : break;
1197 :
1198 24 : case OFTTime:
1199 48 : builder = std::make_shared<arrow::Time32Builder>(
1200 72 : arrow::time32(arrow::TimeUnit::MILLI), m_poMemoryPool);
1201 24 : break;
1202 :
1203 95 : case OFTDateTime:
1204 190 : builder = std::make_shared<arrow::TimestampBuilder>(
1205 285 : arrow::timestamp(arrow::TimeUnit::MILLI), m_poMemoryPool);
1206 95 : break;
1207 : }
1208 1368 : m_apoBuilders.emplace_back(builder);
1209 : }
1210 :
1211 778 : for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i, ++nArrowIdx)
1212 : {
1213 386 : std::shared_ptr<arrow::ArrayBuilder> builder;
1214 386 : const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
1215 386 : const auto eGType = poGeomFieldDefn->GetType();
1216 : const int nDim =
1217 386 : 2 + (OGR_GT_HasZ(eGType) ? 1 : 0) + (OGR_GT_HasM(eGType) ? 1 : 0);
1218 :
1219 386 : switch (m_aeGeomEncoding[i])
1220 : {
1221 126 : case OGRArrowGeomEncoding::WKB:
1222 : builder =
1223 126 : std::make_shared<arrow::BinaryBuilder>(m_poMemoryPool);
1224 126 : break;
1225 :
1226 53 : case OGRArrowGeomEncoding::WKT:
1227 : builder =
1228 53 : std::make_shared<arrow::StringBuilder>(m_poMemoryPool);
1229 53 : break;
1230 :
1231 8 : case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
1232 8 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 0);
1233 8 : break;
1234 :
1235 8 : case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
1236 8 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 1);
1237 8 : break;
1238 :
1239 10 : case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
1240 10 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 2);
1241 10 : break;
1242 :
1243 8 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
1244 8 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 1);
1245 8 : break;
1246 :
1247 8 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
1248 8 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 2);
1249 8 : break;
1250 :
1251 10 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
1252 10 : builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 3);
1253 10 : break;
1254 :
1255 34 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
1256 68 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 0,
1257 68 : m_apoBaseStructGeomType[i]);
1258 34 : break;
1259 :
1260 21 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
1261 42 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 1,
1262 42 : m_apoBaseStructGeomType[i]);
1263 21 : break;
1264 :
1265 29 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
1266 58 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 2,
1267 58 : m_apoBaseStructGeomType[i]);
1268 29 : break;
1269 :
1270 21 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
1271 42 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 1,
1272 42 : m_apoBaseStructGeomType[i]);
1273 21 : break;
1274 :
1275 21 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
1276 42 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 2,
1277 42 : m_apoBaseStructGeomType[i]);
1278 21 : break;
1279 :
1280 29 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
1281 58 : builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 3,
1282 58 : m_apoBaseStructGeomType[i]);
1283 29 : break;
1284 :
1285 0 : case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
1286 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
1287 0 : CPLAssert(false);
1288 : break;
1289 : }
1290 :
1291 386 : m_apoBuilders.emplace_back(builder);
1292 :
1293 386 : if (m_bWriteBBoxStruct)
1294 : {
1295 : m_apoBuildersBBOXXMin.emplace_back(
1296 189 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1297 : m_apoBuildersBBOXYMin.emplace_back(
1298 189 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1299 : m_apoBuildersBBOXXMax.emplace_back(
1300 189 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1301 : m_apoBuildersBBOXYMax.emplace_back(
1302 189 : std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1303 : m_apoBuildersBBOXStruct.emplace_back(
1304 378 : std::make_shared<arrow::StructBuilder>(
1305 189 : m_apoFieldsBBOX[i]->type(), m_poMemoryPool,
1306 1701 : std::vector<std::shared_ptr<arrow::ArrayBuilder>>{
1307 189 : m_apoBuildersBBOXXMin.back(),
1308 189 : m_apoBuildersBBOXYMin.back(),
1309 189 : m_apoBuildersBBOXXMax.back(),
1310 1323 : m_apoBuildersBBOXYMax.back()}));
1311 : }
1312 : }
1313 392 : }
1314 :
1315 : /************************************************************************/
1316 : /* castToFloatDown() */
1317 : /************************************************************************/
1318 :
1319 : // Cf https://github.com/sqlite/sqlite/blob/90e4a3b7fcdf63035d6f35eb44d11ff58ff4b068/ext/rtree/rtree.c#L2993C1-L2995C3
1320 : /*
1321 : ** Rounding constants for float->double conversion.
1322 : */
1323 : #define RNDTOWARDS (1.0 - 1.0 / 8388608.0) /* Round towards zero */
1324 : #define RNDAWAY (1.0 + 1.0 / 8388608.0) /* Round away from zero */
1325 :
1326 : /*
1327 : ** Convert an sqlite3_value into an RtreeValue (presumably a float)
1328 : ** while taking care to round toward negative or positive, respectively.
1329 : */
1330 3490 : static float castToFloatDown(double d)
1331 : {
1332 3490 : float f = static_cast<float>(d);
1333 3490 : if (f > d)
1334 : {
1335 12 : f = static_cast<float>(d * (d < 0 ? RNDAWAY : RNDTOWARDS));
1336 : }
1337 3490 : return f;
1338 : }
1339 :
1340 3490 : static float castToFloatUp(double d)
1341 : {
1342 3490 : float f = static_cast<float>(d);
1343 3490 : if (f < d)
1344 : {
1345 7 : f = static_cast<float>(d * (d < 0 ? RNDTOWARDS : RNDAWAY));
1346 : }
1347 3490 : return f;
1348 : }
1349 :
1350 : /************************************************************************/
1351 : /* GeoArrowLineBuilder() */
1352 : /************************************************************************/
1353 :
1354 : template <class PointBuilderType>
1355 516 : static OGRErr GeoArrowLineBuilder(const OGRLineString *poLS,
1356 : PointBuilderType *poPointBuilder,
1357 : arrow::DoubleBuilder *poXBuilder,
1358 : arrow::DoubleBuilder *poYBuilder,
1359 : arrow::DoubleBuilder *poZBuilder,
1360 : arrow::DoubleBuilder *poMBuilder)
1361 : {
1362 2360 : for (int j = 0; j < poLS->getNumPoints(); ++j)
1363 : {
1364 1844 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1365 1844 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poXBuilder->Append(poLS->getX(j)));
1366 1844 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poYBuilder->Append(poLS->getY(j)));
1367 1844 : if (poZBuilder)
1368 540 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poZBuilder->Append(poLS->getZ(j)));
1369 1844 : if (poMBuilder)
1370 220 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMBuilder->Append(poLS->getM(j)));
1371 : }
1372 516 : return OGRERR_NONE;
1373 : }
1374 :
1375 : /************************************************************************/
1376 : /* BuildGeometry() */
1377 : /************************************************************************/
1378 :
1379 3592 : inline OGRErr OGRArrowWriterLayer::BuildGeometry(OGRGeometry *poGeom,
1380 : int iGeomField,
1381 : arrow::ArrayBuilder *poBuilder)
1382 : {
1383 3592 : const auto eGType = poGeom ? poGeom->getGeometryType() : wkbNone;
1384 : const auto eColumnGType =
1385 3592 : m_poFeatureDefn->GetGeomFieldDefn(iGeomField)->GetType();
1386 3592 : const bool bHasZ = CPL_TO_BOOL(OGR_GT_HasZ(eColumnGType));
1387 3592 : const bool bHasM = CPL_TO_BOOL(OGR_GT_HasM(eColumnGType));
1388 3592 : const bool bIsEmpty = poGeom != nullptr && poGeom->IsEmpty();
1389 3592 : OGREnvelope3D oEnvelope;
1390 3592 : if (poGeom != nullptr && !bIsEmpty)
1391 : {
1392 2093 : if (poGeom->Is3D())
1393 : {
1394 266 : poGeom->getEnvelope(&oEnvelope);
1395 266 : m_aoEnvelopes[iGeomField].Merge(oEnvelope);
1396 : }
1397 : else
1398 : {
1399 1827 : poGeom->getEnvelope(static_cast<OGREnvelope *>(&oEnvelope));
1400 1827 : m_aoEnvelopes[iGeomField].Merge(oEnvelope);
1401 : }
1402 2093 : m_oSetWrittenGeometryTypes[iGeomField].insert(eGType);
1403 : }
1404 :
1405 3592 : if (m_bWriteBBoxStruct)
1406 : {
1407 2889 : if (poGeom && !bIsEmpty)
1408 : {
1409 1702 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1410 : m_apoBuildersBBOXXMin[iGeomField]->Append(
1411 : castToFloatDown(oEnvelope.MinX)));
1412 1702 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1413 : m_apoBuildersBBOXYMin[iGeomField]->Append(
1414 : castToFloatDown(oEnvelope.MinY)));
1415 1702 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1416 : m_apoBuildersBBOXXMax[iGeomField]->Append(
1417 : castToFloatUp(oEnvelope.MaxX)));
1418 1702 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1419 : m_apoBuildersBBOXYMax[iGeomField]->Append(
1420 : castToFloatUp(oEnvelope.MaxY)));
1421 1702 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1422 : m_apoBuildersBBOXStruct[iGeomField]->Append());
1423 : }
1424 : else
1425 : {
1426 1187 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1427 : m_apoBuildersBBOXStruct[iGeomField]->AppendNull());
1428 : }
1429 : }
1430 :
1431 3592 : if (poGeom == nullptr)
1432 : {
1433 3831 : if (m_aeGeomEncoding[iGeomField] ==
1434 1285 : OGRArrowGeomEncoding::GEOARROW_FSL_POINT &&
1435 1285 : GetDriverUCName() == "PARQUET")
1436 : {
1437 : // For some reason, Parquet doesn't support a NULL FixedSizeList
1438 : // on reading
1439 4 : auto poPointBuilder =
1440 : static_cast<arrow::FixedSizeListBuilder *>(poBuilder);
1441 4 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1442 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1443 4 : poPointBuilder->value_builder());
1444 4 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1445 : std::numeric_limits<double>::quiet_NaN()));
1446 4 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1447 : std::numeric_limits<double>::quiet_NaN()));
1448 4 : if (bHasZ)
1449 2 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1450 : std::numeric_limits<double>::quiet_NaN()));
1451 4 : if (bHasM)
1452 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1453 : std::numeric_limits<double>::quiet_NaN()));
1454 : }
1455 : else
1456 : {
1457 1273 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1458 : }
1459 :
1460 1277 : return OGRERR_NONE;
1461 : }
1462 :
1463 : // The following checks are only valid for GeoArrow encoding
1464 3297 : if (m_aeGeomEncoding[iGeomField] != OGRArrowGeomEncoding::WKB &&
1465 982 : m_aeGeomEncoding[iGeomField] != OGRArrowGeomEncoding::WKT)
1466 : {
1467 862 : if ((!bIsEmpty && eGType != eColumnGType) ||
1468 188 : (bIsEmpty && wkbFlatten(eGType) != wkbFlatten(eColumnGType)))
1469 : {
1470 6 : CPLError(CE_Warning, CPLE_AppDefined,
1471 : "Geometry of type %s found, whereas %s is expected. "
1472 : "Writing null geometry",
1473 : OGRGeometryTypeToName(eGType),
1474 : OGRGeometryTypeToName(eColumnGType));
1475 6 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1476 :
1477 6 : return OGRERR_NONE;
1478 : }
1479 : }
1480 :
1481 2309 : switch (m_aeGeomEncoding[iGeomField])
1482 : {
1483 1333 : case OGRArrowGeomEncoding::WKB:
1484 : {
1485 0 : std::unique_ptr<OGRGeometry> poGeomModified;
1486 1333 : if (OGR_GT_HasM(eGType) && !OGR_GT_HasM(eColumnGType))
1487 : {
1488 : static bool bHasWarned = false;
1489 0 : if (!bHasWarned)
1490 : {
1491 0 : CPLError(CE_Warning, CPLE_AppDefined,
1492 : "Removing M component from geometry");
1493 0 : bHasWarned = true;
1494 : }
1495 0 : poGeomModified.reset(poGeom->clone());
1496 0 : poGeomModified->setMeasured(false);
1497 0 : poGeom = poGeomModified.get();
1498 : }
1499 1333 : FixupGeometryBeforeWriting(poGeom);
1500 1333 : const auto nSize = poGeom->WkbSize();
1501 1333 : if (nSize < INT_MAX)
1502 : {
1503 1333 : m_abyBuffer.resize(nSize);
1504 1333 : poGeom->exportToWkb(wkbNDR, &m_abyBuffer[0], wkbVariantIso);
1505 1333 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1506 : static_cast<arrow::BinaryBuilder *>(poBuilder)->Append(
1507 : m_abyBuffer.data(),
1508 : static_cast<int>(m_abyBuffer.size())));
1509 : }
1510 : else
1511 : {
1512 0 : CPLError(CE_Warning, CPLE_AppDefined,
1513 : "Too big geometry. "
1514 : "Writing null geometry");
1515 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1516 : }
1517 1333 : break;
1518 : }
1519 :
1520 308 : case OGRArrowGeomEncoding::WKT:
1521 : {
1522 308 : OGRWktOptions options;
1523 308 : options.variant = wkbVariantIso;
1524 308 : if (m_nWKTCoordinatePrecision >= 0)
1525 : {
1526 0 : options.format = OGRWktFormat::F;
1527 0 : options.xyPrecision = m_nWKTCoordinatePrecision;
1528 0 : options.zPrecision = m_nWKTCoordinatePrecision;
1529 0 : options.mPrecision = m_nWKTCoordinatePrecision;
1530 : }
1531 308 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1532 : static_cast<arrow::StringBuilder *>(poBuilder)->Append(
1533 : poGeom->exportToWkt(options)));
1534 308 : break;
1535 : }
1536 :
1537 20 : case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
1538 : {
1539 20 : const auto poPoint = poGeom->toPoint();
1540 20 : auto poPointBuilder =
1541 : static_cast<arrow::FixedSizeListBuilder *>(poBuilder);
1542 20 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1543 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1544 20 : poPointBuilder->value_builder());
1545 20 : if (bIsEmpty)
1546 : {
1547 8 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1548 : std::numeric_limits<double>::quiet_NaN()));
1549 8 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1550 : std::numeric_limits<double>::quiet_NaN()));
1551 8 : if (bHasZ)
1552 4 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1553 : std::numeric_limits<double>::quiet_NaN()));
1554 8 : if (bHasM)
1555 2 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1556 : std::numeric_limits<double>::quiet_NaN()));
1557 : }
1558 : else
1559 : {
1560 12 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1561 : poValueBuilder->Append(poPoint->getX()));
1562 12 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1563 : poValueBuilder->Append(poPoint->getY()));
1564 12 : if (bHasZ)
1565 6 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1566 : poValueBuilder->Append(poPoint->getZ()));
1567 12 : if (bHasM)
1568 2 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1569 : poValueBuilder->Append(poPoint->getM()));
1570 : }
1571 20 : break;
1572 : }
1573 :
1574 : #define GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder) \
1575 : auto poXBuilder = \
1576 : static_cast<arrow::DoubleBuilder *>(poPointBuilder->field_builder(0)); \
1577 : auto poYBuilder = \
1578 : static_cast<arrow::DoubleBuilder *>(poPointBuilder->field_builder(1)); \
1579 : int iSubField = 2; \
1580 : arrow::DoubleBuilder *poZBuilder = nullptr; \
1581 : if (bHasZ) \
1582 : { \
1583 : poZBuilder = static_cast<arrow::DoubleBuilder *>( \
1584 : poPointBuilder->field_builder(iSubField)); \
1585 : ++iSubField; \
1586 : } \
1587 : arrow::DoubleBuilder *poMBuilder = nullptr; \
1588 : if (bHasM) \
1589 : { \
1590 : poMBuilder = static_cast<arrow::DoubleBuilder *>( \
1591 : poPointBuilder->field_builder(iSubField)); \
1592 : } \
1593 : do \
1594 : { \
1595 : } while (0)
1596 :
1597 85 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
1598 : {
1599 85 : const auto poPoint = poGeom->toPoint();
1600 85 : auto poPointBuilder =
1601 : static_cast<arrow::StructBuilder *>(poBuilder);
1602 85 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1603 85 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1604 :
1605 85 : if (bIsEmpty)
1606 : {
1607 20 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poXBuilder->Append(
1608 : std::numeric_limits<double>::quiet_NaN()));
1609 20 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poYBuilder->Append(
1610 : std::numeric_limits<double>::quiet_NaN()));
1611 : }
1612 : else
1613 : {
1614 65 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1615 : poXBuilder->Append(poPoint->getX()));
1616 65 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1617 : poYBuilder->Append(poPoint->getY()));
1618 : }
1619 85 : if (poZBuilder)
1620 : {
1621 28 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poZBuilder->Append(
1622 : bIsEmpty ? std::numeric_limits<double>::quiet_NaN()
1623 : : poPoint->getZ()));
1624 : }
1625 85 : if (poMBuilder)
1626 : {
1627 4 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMBuilder->Append(
1628 : bIsEmpty ? std::numeric_limits<double>::quiet_NaN()
1629 : : poPoint->getM()));
1630 : }
1631 85 : break;
1632 : }
1633 :
1634 20 : case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
1635 : {
1636 20 : const auto poLS = poGeom->toLineString();
1637 20 : auto poListBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1638 : auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1639 20 : poListBuilder->value_builder());
1640 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1641 20 : poPointBuilder->value_builder());
1642 :
1643 20 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
1644 20 : OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1645 : poLS, poPointBuilder, poValueBuilder, poValueBuilder,
1646 : bHasZ ? poValueBuilder : nullptr,
1647 : bHasM ? poValueBuilder : nullptr));
1648 20 : break;
1649 : }
1650 :
1651 57 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
1652 : {
1653 57 : const auto poLS = poGeom->toLineString();
1654 57 : auto poListBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1655 : auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1656 57 : poListBuilder->value_builder());
1657 57 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1658 :
1659 57 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
1660 57 : OGR_ARROW_PROPAGATE_OGRERR(
1661 : GeoArrowLineBuilder(poLS, poPointBuilder, poXBuilder,
1662 : poYBuilder, poZBuilder, poMBuilder));
1663 57 : break;
1664 : }
1665 :
1666 32 : case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
1667 : {
1668 32 : const auto poPolygon = poGeom->toPolygon();
1669 32 : auto poPolygonBuilder =
1670 : static_cast<arrow::ListBuilder *>(poBuilder);
1671 : auto poRingBuilder = static_cast<arrow::ListBuilder *>(
1672 32 : poPolygonBuilder->value_builder());
1673 : auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1674 32 : poRingBuilder->value_builder());
1675 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1676 32 : poPointBuilder->value_builder());
1677 32 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPolygonBuilder->Append());
1678 62 : for (const auto *poRing : *poPolygon)
1679 : {
1680 30 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poRingBuilder->Append());
1681 30 : OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1682 : poRing, poPointBuilder, poValueBuilder, poValueBuilder,
1683 : bHasZ ? poValueBuilder : nullptr,
1684 : bHasM ? poValueBuilder : nullptr));
1685 : }
1686 32 : break;
1687 : }
1688 :
1689 93 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
1690 : {
1691 93 : const auto poPolygon = poGeom->toPolygon();
1692 93 : auto poPolygonBuilder =
1693 : static_cast<arrow::ListBuilder *>(poBuilder);
1694 : auto poRingBuilder = static_cast<arrow::ListBuilder *>(
1695 93 : poPolygonBuilder->value_builder());
1696 : auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1697 93 : poRingBuilder->value_builder());
1698 93 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1699 :
1700 93 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPolygonBuilder->Append());
1701 178 : for (const auto *poRing : *poPolygon)
1702 : {
1703 85 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poRingBuilder->Append());
1704 85 : OGR_ARROW_PROPAGATE_OGRERR(
1705 : GeoArrowLineBuilder(poRing, poPointBuilder, poXBuilder,
1706 : poYBuilder, poZBuilder, poMBuilder));
1707 : }
1708 93 : break;
1709 : }
1710 :
1711 32 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
1712 : {
1713 32 : const auto poMultiPoint = poGeom->toMultiPoint();
1714 32 : auto poListBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1715 : auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1716 32 : poListBuilder->value_builder());
1717 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1718 32 : poPointBuilder->value_builder());
1719 32 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
1720 88 : for (const auto *poPoint : *poMultiPoint)
1721 : {
1722 56 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1723 56 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1724 : poValueBuilder->Append(poPoint->getX()));
1725 56 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1726 : poValueBuilder->Append(poPoint->getY()));
1727 56 : if (bHasZ)
1728 28 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1729 : poValueBuilder->Append(poPoint->getZ()));
1730 56 : if (bHasM)
1731 18 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1732 : poValueBuilder->Append(poPoint->getM()));
1733 : }
1734 32 : break;
1735 : }
1736 :
1737 81 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
1738 : {
1739 81 : const auto poMultiPoint = poGeom->toMultiPoint();
1740 81 : auto poListBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1741 : auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1742 81 : poListBuilder->value_builder());
1743 81 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1744 :
1745 81 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
1746 198 : for (const auto *poPoint : *poMultiPoint)
1747 : {
1748 117 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1749 117 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1750 : poXBuilder->Append(poPoint->getX()));
1751 117 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1752 : poYBuilder->Append(poPoint->getY()));
1753 117 : if (poZBuilder)
1754 58 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1755 : poZBuilder->Append(poPoint->getZ()));
1756 117 : if (poMBuilder)
1757 18 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1758 : poMBuilder->Append(poPoint->getM()));
1759 : }
1760 81 : break;
1761 : }
1762 :
1763 28 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
1764 : {
1765 28 : const auto poMLS = poGeom->toMultiLineString();
1766 28 : auto poMLSBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1767 : auto poLSBuilder = static_cast<arrow::ListBuilder *>(
1768 28 : poMLSBuilder->value_builder());
1769 : auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1770 28 : poLSBuilder->value_builder());
1771 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1772 28 : poPointBuilder->value_builder());
1773 28 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMLSBuilder->Append());
1774 60 : for (const auto *poLS : *poMLS)
1775 : {
1776 32 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poLSBuilder->Append());
1777 32 : OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1778 : poLS, poPointBuilder, poValueBuilder, poValueBuilder,
1779 : bHasZ ? poValueBuilder : nullptr,
1780 : bHasM ? poValueBuilder : nullptr));
1781 : }
1782 28 : break;
1783 : }
1784 :
1785 77 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
1786 : {
1787 77 : const auto poMLS = poGeom->toMultiLineString();
1788 77 : auto poMLSBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1789 : auto poLSBuilder = static_cast<arrow::ListBuilder *>(
1790 77 : poMLSBuilder->value_builder());
1791 : auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1792 77 : poLSBuilder->value_builder());
1793 77 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1794 :
1795 77 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMLSBuilder->Append());
1796 170 : for (const auto *poLS : *poMLS)
1797 : {
1798 93 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poLSBuilder->Append());
1799 93 : OGR_ARROW_PROPAGATE_OGRERR(
1800 : GeoArrowLineBuilder(poLS, poPointBuilder, poXBuilder,
1801 : poYBuilder, poZBuilder, poMBuilder));
1802 : }
1803 77 : break;
1804 : }
1805 :
1806 38 : case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
1807 : {
1808 38 : const auto poMPoly = poGeom->toMultiPolygon();
1809 38 : auto poMPolyBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1810 : auto poPolyBuilder = static_cast<arrow::ListBuilder *>(
1811 38 : poMPolyBuilder->value_builder());
1812 : auto poRingBuilder = static_cast<arrow::ListBuilder *>(
1813 38 : poPolyBuilder->value_builder());
1814 : auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1815 38 : poRingBuilder->value_builder());
1816 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1817 38 : poPointBuilder->value_builder());
1818 38 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMPolyBuilder->Append());
1819 82 : for (const auto *poPolygon : *poMPoly)
1820 : {
1821 44 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPolyBuilder->Append());
1822 98 : for (const auto *poRing : *poPolygon)
1823 : {
1824 54 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poRingBuilder->Append());
1825 54 : OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1826 : poRing, poPointBuilder, poValueBuilder, poValueBuilder,
1827 : bHasZ ? poValueBuilder : nullptr,
1828 : bHasM ? poValueBuilder : nullptr));
1829 : }
1830 : }
1831 38 : break;
1832 : }
1833 :
1834 105 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
1835 : {
1836 105 : const auto poMPoly = poGeom->toMultiPolygon();
1837 105 : auto poMPolyBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1838 : auto poPolyBuilder = static_cast<arrow::ListBuilder *>(
1839 105 : poMPolyBuilder->value_builder());
1840 : auto poRingBuilder = static_cast<arrow::ListBuilder *>(
1841 105 : poPolyBuilder->value_builder());
1842 : auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1843 105 : poRingBuilder->value_builder());
1844 105 : GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1845 :
1846 105 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poMPolyBuilder->Append());
1847 222 : for (const auto *poPolygon : *poMPoly)
1848 : {
1849 117 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poPolyBuilder->Append());
1850 262 : for (const auto *poRing : *poPolygon)
1851 : {
1852 145 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poRingBuilder->Append());
1853 145 : OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1854 : poRing, poPointBuilder, poXBuilder, poYBuilder,
1855 : poZBuilder, poMBuilder));
1856 : }
1857 : }
1858 105 : break;
1859 : }
1860 :
1861 0 : case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
1862 : case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
1863 : {
1864 0 : CPLAssert(false);
1865 : break;
1866 : }
1867 : }
1868 :
1869 2309 : return OGRERR_NONE;
1870 : }
1871 :
1872 : /************************************************************************/
1873 : /* ICreateFeature() */
1874 : /************************************************************************/
1875 :
1876 3138 : inline OGRErr OGRArrowWriterLayer::ICreateFeature(OGRFeature *poFeature)
1877 : {
1878 3138 : if (m_poSchema == nullptr)
1879 : {
1880 234 : CreateSchema();
1881 : }
1882 :
1883 3138 : if (m_apoBuilders.empty())
1884 : {
1885 271 : if (!m_apoFieldsFromArrowSchema.empty())
1886 : {
1887 0 : CPLError(CE_Failure, CPLE_NotSupported,
1888 : "ICreateFeature() cannot be used after "
1889 : "CreateFieldFromArrowSchema()");
1890 0 : return OGRERR_FAILURE;
1891 : }
1892 271 : CreateArrayBuilders();
1893 : }
1894 :
1895 : // First pass to check not-null constraints as Arrow doesn't seem
1896 : // to do that on the writing side. But such files can't be read.
1897 3138 : const int nFieldCount = m_poFeatureDefn->GetFieldCount();
1898 8373 : for (int i = 0; i < nFieldCount; ++i)
1899 : {
1900 5236 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
1901 5238 : if (!poFieldDefn->IsNullable() &&
1902 2 : !poFeature->IsFieldSetAndNotNullUnsafe(i))
1903 : {
1904 1 : CPLError(CE_Failure, CPLE_AppDefined,
1905 : "Null value found in non-nullable field %s",
1906 : poFieldDefn->GetNameRef());
1907 1 : return OGRERR_FAILURE;
1908 : }
1909 : }
1910 :
1911 3137 : const int nGeomFieldCount = m_poFeatureDefn->GetGeomFieldCount();
1912 6380 : for (int i = 0; i < nGeomFieldCount; ++i)
1913 : {
1914 3243 : const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
1915 3275 : if (!poGeomFieldDefn->IsNullable() &&
1916 32 : poFeature->GetGeomFieldRef(i) == nullptr)
1917 : {
1918 0 : CPLError(CE_Failure, CPLE_AppDefined,
1919 : "Null value found in non-nullable geometry field %s",
1920 : poGeomFieldDefn->GetNameRef());
1921 0 : return OGRERR_FAILURE;
1922 : }
1923 : }
1924 :
1925 : // Write FID, if FID column present
1926 3137 : int nArrowIdx = 0;
1927 3137 : if (!m_osFIDColumn.empty())
1928 : {
1929 2258 : int64_t nFID = poFeature->GetFID();
1930 2258 : if (nFID == OGRNullFID)
1931 : {
1932 36 : nFID = m_nFeatureCount;
1933 36 : poFeature->SetFID(nFID);
1934 : }
1935 : auto poBuilder =
1936 2258 : static_cast<arrow::Int64Builder *>(m_apoBuilders[0].get());
1937 2258 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->Append(nFID));
1938 2258 : nArrowIdx++;
1939 : }
1940 :
1941 : // Write attributes
1942 8372 : for (int i = 0; i < nFieldCount; ++i, ++nArrowIdx)
1943 : {
1944 5235 : auto poBuilder = m_apoBuilders[nArrowIdx].get();
1945 5235 : if (!poFeature->IsFieldSetAndNotNullUnsafe(i))
1946 : {
1947 971 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1948 971 : continue;
1949 : }
1950 :
1951 4264 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
1952 4264 : const auto eSubDT = poFieldDefn->GetSubType();
1953 4264 : switch (poFieldDefn->GetType())
1954 : {
1955 2453 : case OFTInteger:
1956 2453 : if (eSubDT == OFSTBoolean)
1957 16 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1958 : static_cast<arrow::BooleanBuilder *>(poBuilder)->Append(
1959 : poFeature->GetFieldAsIntegerUnsafe(i) != 0));
1960 2437 : else if (eSubDT == OFSTInt16)
1961 16 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1962 : static_cast<arrow::Int16Builder *>(poBuilder)->Append(
1963 : static_cast<int16_t>(
1964 : poFeature->GetFieldAsIntegerUnsafe(i))));
1965 : else
1966 2421 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1967 : static_cast<arrow::Int32Builder *>(poBuilder)->Append(
1968 : poFeature->GetFieldAsIntegerUnsafe(i)));
1969 2453 : break;
1970 :
1971 158 : case OFTInteger64:
1972 158 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1973 : static_cast<arrow::Int64Builder *>(poBuilder)->Append(
1974 : static_cast<int64_t>(
1975 : poFeature->GetFieldAsInteger64Unsafe(i))));
1976 158 : break;
1977 :
1978 221 : case OFTReal:
1979 : {
1980 221 : const auto arrowType = m_poSchema->fields()[nArrowIdx]->type();
1981 221 : const double dfVal = poFeature->GetFieldAsDoubleUnsafe(i);
1982 221 : if (arrowType->id() == arrow::Type::DECIMAL128)
1983 : {
1984 : auto res = arrow::Decimal128::FromReal(
1985 : dfVal, poFieldDefn->GetWidth(),
1986 52 : poFieldDefn->GetPrecision());
1987 52 : if (res.ok())
1988 : {
1989 52 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
1990 : static_cast<arrow::Decimal128Builder *>(poBuilder)
1991 : ->Append(*res));
1992 : }
1993 : else
1994 : {
1995 0 : CPLError(CE_Warning, CPLE_AppDefined,
1996 : "Cannot parse %.18g as a %d.%d decimal", dfVal,
1997 : poFieldDefn->GetWidth(),
1998 : poFieldDefn->GetPrecision());
1999 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
2000 : }
2001 : }
2002 169 : else if (arrowType->id() == arrow::Type::DECIMAL256)
2003 : {
2004 : auto res = arrow::Decimal256::FromReal(
2005 : dfVal, poFieldDefn->GetWidth(),
2006 0 : poFieldDefn->GetPrecision());
2007 0 : if (res.ok())
2008 : {
2009 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2010 : static_cast<arrow::Decimal256Builder *>(poBuilder)
2011 : ->Append(*res));
2012 : }
2013 : else
2014 : {
2015 0 : CPLError(CE_Warning, CPLE_AppDefined,
2016 : "Cannot parse %.18g as a %d.%d decimal", dfVal,
2017 : poFieldDefn->GetWidth(),
2018 : poFieldDefn->GetPrecision());
2019 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
2020 : }
2021 : }
2022 169 : else if (eSubDT == OFSTFloat32)
2023 : {
2024 28 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2025 : static_cast<arrow::FloatBuilder *>(poBuilder)->Append(
2026 : static_cast<float>(dfVal)));
2027 : }
2028 : else
2029 : {
2030 141 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2031 : static_cast<arrow::DoubleBuilder *>(poBuilder)->Append(
2032 : dfVal));
2033 : }
2034 221 : break;
2035 : }
2036 :
2037 524 : case OFTString:
2038 : case OFTWideString:
2039 524 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2040 : static_cast<arrow::StringBuilder *>(poBuilder)->Append(
2041 : poFeature->GetFieldAsStringUnsafe(i)));
2042 524 : break;
2043 :
2044 62 : case OFTBinary:
2045 : {
2046 62 : int nSize = 0;
2047 62 : const auto pData = poFeature->GetFieldAsBinary(i, &nSize);
2048 62 : if (poFieldDefn->GetWidth() != 0)
2049 : {
2050 20 : if (poFieldDefn->GetWidth() != nSize)
2051 : {
2052 0 : CPLError(
2053 : CE_Warning, CPLE_AppDefined,
2054 : "Cannot write field %s. Got %d bytes, expected %d",
2055 : poFieldDefn->GetNameRef(), nSize,
2056 : poFieldDefn->GetWidth());
2057 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
2058 : }
2059 : else
2060 : {
2061 20 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2062 : static_cast<arrow::FixedSizeBinaryBuilder *>(
2063 : poBuilder)
2064 : ->Append(pData));
2065 : }
2066 : }
2067 : else
2068 42 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2069 : static_cast<arrow::BinaryBuilder *>(poBuilder)->Append(
2070 : pData, nSize));
2071 62 : break;
2072 : }
2073 :
2074 216 : case OFTIntegerList:
2075 : {
2076 216 : auto poListBuilder =
2077 : static_cast<arrow::ListBuilder *>(poBuilder);
2078 216 : if (eSubDT == OFSTBoolean)
2079 : {
2080 36 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2081 : auto poValueBuilder = static_cast<arrow::BooleanBuilder *>(
2082 36 : poListBuilder->value_builder());
2083 36 : int nValues = 0;
2084 : const auto panValues =
2085 36 : poFeature->GetFieldAsIntegerList(i, &nValues);
2086 108 : for (int j = 0; j < nValues; ++j)
2087 72 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2088 : poValueBuilder->Append(panValues[j] != 0));
2089 : }
2090 180 : else if (eSubDT == OFSTInt16)
2091 : {
2092 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2093 : auto poValueBuilder = static_cast<arrow::Int16Builder *>(
2094 0 : poListBuilder->value_builder());
2095 0 : int nValues = 0;
2096 : const auto panValues =
2097 0 : poFeature->GetFieldAsIntegerList(i, &nValues);
2098 0 : for (int j = 0; j < nValues; ++j)
2099 0 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
2100 : static_cast<int16_t>(panValues[j])));
2101 : }
2102 : else
2103 : {
2104 180 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2105 : auto poValueBuilder = static_cast<arrow::Int32Builder *>(
2106 180 : poListBuilder->value_builder());
2107 180 : int nValues = 0;
2108 : const auto panValues =
2109 180 : poFeature->GetFieldAsIntegerList(i, &nValues);
2110 540 : for (int j = 0; j < nValues; ++j)
2111 360 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2112 : poValueBuilder->Append(panValues[j]));
2113 : }
2114 216 : break;
2115 : }
2116 :
2117 92 : case OFTInteger64List:
2118 : {
2119 92 : auto poListBuilder =
2120 : static_cast<arrow::ListBuilder *>(poBuilder);
2121 92 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2122 : auto poValueBuilder = static_cast<arrow::Int64Builder *>(
2123 92 : poListBuilder->value_builder());
2124 92 : int nValues = 0;
2125 : const auto panValues =
2126 92 : poFeature->GetFieldAsInteger64List(i, &nValues);
2127 292 : for (int j = 0; j < nValues; ++j)
2128 200 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
2129 : static_cast<int64_t>(panValues[j])));
2130 92 : break;
2131 : }
2132 :
2133 152 : case OFTRealList:
2134 : {
2135 152 : auto poListBuilder =
2136 : static_cast<arrow::ListBuilder *>(poBuilder);
2137 152 : if (eSubDT == OFSTFloat32)
2138 : {
2139 48 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2140 : auto poValueBuilder = static_cast<arrow::FloatBuilder *>(
2141 48 : poListBuilder->value_builder());
2142 48 : int nValues = 0;
2143 : const auto padfValues =
2144 48 : poFeature->GetFieldAsDoubleList(i, &nValues);
2145 144 : for (int j = 0; j < nValues; ++j)
2146 96 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
2147 : static_cast<float>(padfValues[j])));
2148 : }
2149 : else
2150 : {
2151 104 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2152 : auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
2153 104 : poListBuilder->value_builder());
2154 104 : int nValues = 0;
2155 : const auto padfValues =
2156 104 : poFeature->GetFieldAsDoubleList(i, &nValues);
2157 280 : for (int j = 0; j < nValues; ++j)
2158 176 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2159 : poValueBuilder->Append(padfValues[j]));
2160 : }
2161 152 : break;
2162 : }
2163 :
2164 52 : case OFTStringList:
2165 : case OFTWideStringList:
2166 : {
2167 52 : auto poListBuilder =
2168 : static_cast<arrow::ListBuilder *>(poBuilder);
2169 52 : OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2170 : auto poValueBuilder = static_cast<arrow::StringBuilder *>(
2171 52 : poListBuilder->value_builder());
2172 52 : const auto papszValues = poFeature->GetFieldAsStringList(i);
2173 132 : for (int j = 0; papszValues && papszValues[j]; ++j)
2174 80 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2175 : poValueBuilder->Append(papszValues[j]));
2176 52 : break;
2177 : }
2178 :
2179 109 : case OFTDate:
2180 : {
2181 : int nYear, nMonth, nDay, nHour, nMinute;
2182 : float fSec;
2183 : int nTZFlag;
2184 109 : poFeature->GetFieldAsDateTime(i, &nYear, &nMonth, &nDay, &nHour,
2185 : &nMinute, &fSec, &nTZFlag);
2186 : struct tm brokenDown;
2187 109 : memset(&brokenDown, 0, sizeof(brokenDown));
2188 109 : brokenDown.tm_year = nYear - 1900;
2189 109 : brokenDown.tm_mon = nMonth - 1;
2190 109 : brokenDown.tm_mday = nDay;
2191 109 : GIntBig nVal = CPLYMDHMSToUnixTime(&brokenDown);
2192 109 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2193 : static_cast<arrow::Date32Builder *>(poBuilder)->Append(
2194 : static_cast<int>(nVal / 86400)));
2195 109 : break;
2196 : }
2197 :
2198 36 : case OFTTime:
2199 : {
2200 : int nYear, nMonth, nDay, nHour, nMinute;
2201 : float fSec;
2202 : int nTZFlag;
2203 36 : poFeature->GetFieldAsDateTime(i, &nYear, &nMonth, &nDay, &nHour,
2204 : &nMinute, &fSec, &nTZFlag);
2205 36 : int nVal = nHour * 3600 + nMinute * 60;
2206 36 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2207 : static_cast<arrow::Time32Builder *>(poBuilder)->Append(
2208 : static_cast<int>(
2209 : (static_cast<double>(nVal) + fSec) * 1000 + 0.5)));
2210 36 : break;
2211 : }
2212 :
2213 189 : case OFTDateTime:
2214 : {
2215 : int nYear, nMonth, nDay, nHour, nMinute;
2216 : float fSec;
2217 : int nTZFlag;
2218 189 : poFeature->GetFieldAsDateTime(i, &nYear, &nMonth, &nDay, &nHour,
2219 : &nMinute, &fSec, &nTZFlag);
2220 : struct tm brokenDown;
2221 189 : memset(&brokenDown, 0, sizeof(brokenDown));
2222 189 : brokenDown.tm_year = nYear - 1900;
2223 189 : brokenDown.tm_mon = nMonth - 1;
2224 189 : brokenDown.tm_mday = nDay;
2225 189 : brokenDown.tm_hour = nHour;
2226 189 : brokenDown.tm_min = nMinute;
2227 189 : brokenDown.tm_sec = 0;
2228 189 : GIntBig nVal = CPLYMDHMSToUnixTime(&brokenDown);
2229 306 : if (!IsFileWriterCreated() &&
2230 117 : m_anTZFlag[i] != OGR_TZFLAG_UNKNOWN)
2231 : {
2232 59 : if (m_anTZFlag[i] == TZFLAG_UNINITIALIZED)
2233 35 : m_anTZFlag[i] = nTZFlag;
2234 24 : else if (m_anTZFlag[i] != nTZFlag)
2235 : {
2236 0 : if (m_anTZFlag[i] >= OGR_TZFLAG_MIXED_TZ &&
2237 0 : nTZFlag >= OGR_TZFLAG_MIXED_TZ)
2238 : {
2239 0 : m_anTZFlag[i] =
2240 : OGR_TZFLAG_MIXED_TZ; // harmonize on UTC ultimately
2241 : }
2242 : else
2243 : {
2244 0 : CPLError(CE_Warning, CPLE_AppDefined,
2245 : "Field %s contains a mix of "
2246 : "timezone-aware and local/without "
2247 : "timezone values.",
2248 : poFieldDefn->GetNameRef());
2249 0 : m_anTZFlag[i] = OGR_TZFLAG_UNKNOWN;
2250 : }
2251 : }
2252 : }
2253 189 : if (nTZFlag > OGR_TZFLAG_MIXED_TZ)
2254 : {
2255 60 : const int nOffsetSec = (nTZFlag - OGR_TZFLAG_UTC) * 15 * 60;
2256 60 : nVal -= nOffsetSec;
2257 : }
2258 189 : OGR_ARROW_RETURN_OGRERR_NOT_OK(
2259 : static_cast<arrow::TimestampBuilder *>(poBuilder)->Append(
2260 : static_cast<int64_t>(
2261 : (static_cast<double>(nVal) + fSec) * 1000 + 0.5)));
2262 189 : break;
2263 : }
2264 : }
2265 : }
2266 :
2267 : // Write geometries
2268 6380 : for (int i = 0; i < nGeomFieldCount; ++i, ++nArrowIdx)
2269 : {
2270 3243 : auto poBuilder = m_apoBuilders[nArrowIdx].get();
2271 3243 : OGRGeometry *poGeom = poFeature->GetGeomFieldRef(i);
2272 3243 : if (BuildGeometry(poGeom, i, poBuilder) != OGRERR_NONE)
2273 0 : return OGRERR_FAILURE;
2274 : }
2275 :
2276 3137 : m_nFeatureCount++;
2277 :
2278 : // Flush the current row group if reaching the limit of rows per group.
2279 3137 : if (!m_apoBuilders.empty() && m_apoBuilders[0]->length() == m_nRowGroupSize)
2280 : {
2281 22 : if (!FlushFeatures())
2282 0 : return OGRERR_FAILURE;
2283 : }
2284 :
2285 3137 : return OGRERR_NONE;
2286 : }
2287 :
2288 : /************************************************************************/
2289 : /* FlushFeatures() */
2290 : /************************************************************************/
2291 :
2292 38 : inline bool OGRArrowWriterLayer::FlushFeatures()
2293 : {
2294 38 : if (m_apoBuilders.empty() || m_apoBuilders[0]->length() == 0)
2295 0 : return true;
2296 :
2297 38 : if (!IsFileWriterCreated())
2298 : {
2299 8 : CreateWriter();
2300 8 : if (!IsFileWriterCreated())
2301 0 : return false;
2302 : }
2303 :
2304 38 : return FlushGroup();
2305 : }
2306 :
2307 : /************************************************************************/
2308 : /* GetFeatureCount() */
2309 : /************************************************************************/
2310 :
2311 1 : inline GIntBig OGRArrowWriterLayer::GetFeatureCount(int bForce)
2312 : {
2313 1 : if (m_poAttrQuery == nullptr && m_poFilterGeom == nullptr)
2314 : {
2315 1 : return m_nFeatureCount;
2316 : }
2317 0 : return OGRLayer::GetFeatureCount(bForce);
2318 : }
2319 :
2320 : /************************************************************************/
2321 : /* TestCapability() */
2322 : /************************************************************************/
2323 :
2324 642 : inline int OGRArrowWriterLayer::TestCapability(const char *pszCap)
2325 : {
2326 642 : if (EQUAL(pszCap, OLCCreateField) || EQUAL(pszCap, OLCCreateGeomField))
2327 27 : return m_poSchema == nullptr;
2328 :
2329 615 : if (EQUAL(pszCap, OLCSequentialWrite))
2330 24 : return true;
2331 :
2332 591 : if (EQUAL(pszCap, OLCFastWriteArrowBatch))
2333 0 : return true;
2334 :
2335 591 : if (EQUAL(pszCap, OLCStringsAsUTF8))
2336 1 : return true;
2337 :
2338 590 : if (EQUAL(pszCap, OLCMeasuredGeometries))
2339 234 : return true;
2340 :
2341 356 : return false;
2342 : }
2343 :
2344 : /************************************************************************/
2345 : /* WriteArrays() */
2346 : /************************************************************************/
2347 :
2348 273 : inline bool OGRArrowWriterLayer::WriteArrays(
2349 : std::function<bool(const std::shared_ptr<arrow::Field> &,
2350 : const std::shared_ptr<arrow::Array> &)>
2351 : postProcessArray)
2352 : {
2353 273 : int nArrowIdx = 0;
2354 273 : int nArrowIdxFirstField = !m_osFIDColumn.empty() ? 1 : 0;
2355 1956 : for (const auto &poBuilder : m_apoBuilders)
2356 : {
2357 1683 : const auto &field = m_poSchema->fields()[nArrowIdx];
2358 :
2359 0 : std::shared_ptr<arrow::Array> array;
2360 1683 : auto status = poBuilder->Finish(&array);
2361 1683 : if (!status.ok())
2362 : {
2363 0 : CPLError(CE_Failure, CPLE_AppDefined,
2364 : "builder::Finish() for field %s failed with %s",
2365 0 : field->name().c_str(), status.message().c_str());
2366 0 : return false;
2367 : }
2368 :
2369 : // CPLDebug("ARROW", "%s", array->ToString().c_str());
2370 :
2371 1683 : const int iCol = nArrowIdx - nArrowIdxFirstField;
2372 1683 : if (iCol >= 0 && iCol < m_poFeatureDefn->GetFieldCount())
2373 : {
2374 1368 : const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(iCol);
2375 1368 : const auto eFieldType = poFieldDefn->GetType();
2376 1368 : if (eFieldType == OFTInteger || eFieldType == OFTInteger64)
2377 : {
2378 231 : const auto &osDomainName = poFieldDefn->GetDomainName();
2379 : const auto oIter =
2380 231 : m_oMapFieldDomainToStringArray.find(osDomainName);
2381 231 : if (oIter != m_oMapFieldDomainToStringArray.end())
2382 : {
2383 : auto result = arrow::DictionaryArray::FromArrays(
2384 12 : field->type(), array, oIter->second);
2385 12 : if (!result.ok())
2386 : {
2387 0 : CPLError(CE_Failure, CPLE_AppDefined,
2388 : "DictionaryArray::FromArrays() for field %s "
2389 : "failed with %s",
2390 0 : field->name().c_str(),
2391 0 : result.status().message().c_str());
2392 0 : return false;
2393 : }
2394 12 : array = *result;
2395 : }
2396 : }
2397 : }
2398 :
2399 1683 : if (!postProcessArray(field, array))
2400 : {
2401 0 : return false;
2402 : }
2403 :
2404 1683 : nArrowIdx++;
2405 : }
2406 :
2407 273 : if (m_bWriteBBoxStruct)
2408 : {
2409 183 : const int nGeomFieldCount = m_poFeatureDefn->GetGeomFieldCount();
2410 365 : for (int i = 0; i < nGeomFieldCount; ++i)
2411 : {
2412 182 : const auto &field = m_apoFieldsBBOX[i];
2413 0 : std::shared_ptr<arrow::Array> array;
2414 182 : auto status = m_apoBuildersBBOXStruct[i]->Finish(&array);
2415 182 : if (!status.ok())
2416 : {
2417 0 : CPLError(CE_Failure, CPLE_AppDefined,
2418 : "builder::Finish() for field %s failed with %s",
2419 0 : field->name().c_str(), status.message().c_str());
2420 0 : return false;
2421 : }
2422 :
2423 182 : if (!postProcessArray(field, array))
2424 : {
2425 0 : return false;
2426 : }
2427 : }
2428 : }
2429 :
2430 273 : return true;
2431 : }
2432 :
2433 : /************************************************************************/
2434 : /* TestBit() */
2435 : /************************************************************************/
2436 :
2437 504 : static inline bool TestBit(const uint8_t *pabyData, size_t nIdx)
2438 : {
2439 504 : return (pabyData[nIdx / 8] & (1 << (nIdx % 8))) != 0;
2440 : }
2441 :
2442 : /************************************************************************/
2443 : /* WriteArrowBatchInternal() */
2444 : /************************************************************************/
2445 :
2446 126 : inline bool OGRArrowWriterLayer::WriteArrowBatchInternal(
2447 : const struct ArrowSchema *schema, struct ArrowArray *array,
2448 : CSLConstList papszOptions,
2449 : std::function<bool(const std::shared_ptr<arrow::RecordBatch> &)> writeBatch)
2450 : {
2451 : #ifdef __COVERITY__
2452 : (void)schema;
2453 : (void)array;
2454 : (void)papszOptions;
2455 : (void)writeBatch;
2456 : CPLError(CE_Failure, CPLE_AppDefined, "Not implemented");
2457 : return false;
2458 : #else
2459 126 : if (m_poSchema == nullptr)
2460 : {
2461 121 : CreateSchema();
2462 : }
2463 :
2464 126 : if (!IsFileWriterCreated())
2465 : {
2466 121 : CreateWriter();
2467 121 : if (!IsFileWriterCreated())
2468 0 : return false;
2469 : }
2470 :
2471 126 : if (m_apoBuilders.empty())
2472 : {
2473 121 : CreateArrayBuilders();
2474 : }
2475 :
2476 126 : const int nGeomFieldCount = m_poFeatureDefn->GetGeomFieldCount();
2477 126 : const int nGeomFieldCountBBoxFields =
2478 126 : m_bWriteBBoxStruct ? nGeomFieldCount : 0;
2479 :
2480 126 : const char *pszFIDName = CSLFetchNameValueDef(
2481 : papszOptions, "FID", OGRLayer::DEFAULT_ARROW_FID_NAME);
2482 : const char *pszSingleGeomFieldName =
2483 126 : CSLFetchNameValue(papszOptions, "GEOMETRY_NAME");
2484 :
2485 : // Sort schema and array children in the same order as m_poSchema.
2486 : // This is needed for non-WKB geometry encoding
2487 252 : std::map<std::string, int> oMapSchemaChildrenNameToIdx;
2488 1644 : for (int i = 0; i < static_cast<int>(schema->n_children); ++i)
2489 : {
2490 1518 : if (cpl::contains(oMapSchemaChildrenNameToIdx,
2491 1518 : schema->children[i]->name))
2492 : {
2493 0 : CPLError(CE_Failure, CPLE_AppDefined,
2494 : "Several fields with same name '%s' found",
2495 0 : schema->children[i]->name);
2496 0 : return false;
2497 : }
2498 1518 : oMapSchemaChildrenNameToIdx[schema->children[i]->name] = i;
2499 :
2500 1518 : if (!pszSingleGeomFieldName && schema->children[i]->metadata)
2501 : {
2502 : const auto oMetadata =
2503 256 : OGRParseArrowMetadata(schema->children[i]->metadata);
2504 128 : const auto oIter = oMetadata.find(ARROW_EXTENSION_NAME_KEY);
2505 262 : if (oIter != oMetadata.end() &&
2506 134 : (oIter->second == EXTENSION_NAME_OGC_WKB ||
2507 8 : oIter->second == EXTENSION_NAME_GEOARROW_WKB))
2508 : {
2509 126 : pszSingleGeomFieldName = schema->children[i]->name;
2510 : }
2511 : }
2512 : }
2513 126 : if (!pszSingleGeomFieldName)
2514 0 : pszSingleGeomFieldName = OGRLayer::DEFAULT_ARROW_GEOMETRY_NAME;
2515 :
2516 126 : std::vector<int> anMapLayerSchemaToArraySchema(m_poSchema->num_fields(),
2517 378 : -1);
2518 : struct ArrowArray fidArray;
2519 : struct ArrowSchema fidSchema;
2520 126 : memset(&fidArray, 0, sizeof(fidArray));
2521 126 : memset(&fidSchema, 0, sizeof(fidSchema));
2522 252 : std::vector<void *> apBuffersFid;
2523 252 : std::vector<int64_t> fids;
2524 :
2525 252 : std::set<int> oSetReferencedFieldsInArraySchema;
2526 0 : const auto DummyFreeArray = [](struct ArrowArray *ptrArray)
2527 0 : { ptrArray->release = nullptr; };
2528 126 : const auto DummyFreeSchema = [](struct ArrowSchema *ptrSchema)
2529 126 : { ptrSchema->release = nullptr; };
2530 126 : bool bRebuildBatch = false;
2531 1642 : for (int i = 0; i < m_poSchema->num_fields() - nGeomFieldCountBBoxFields;
2532 : ++i)
2533 : {
2534 : auto oIter =
2535 1516 : oMapSchemaChildrenNameToIdx.find(m_poSchema->field(i)->name());
2536 1516 : if (oIter == oMapSchemaChildrenNameToIdx.end())
2537 : {
2538 4 : if (m_poSchema->field(i)->name() == m_osFIDColumn)
2539 : {
2540 1 : oIter = oMapSchemaChildrenNameToIdx.find(pszFIDName);
2541 1 : if (oIter == oMapSchemaChildrenNameToIdx.end())
2542 : {
2543 : // If the input data does not contain a FID column, but
2544 : // the output file requires it, creates a default FID column
2545 0 : fidArray.release = DummyFreeArray;
2546 0 : fidArray.n_buffers = 2;
2547 0 : apBuffersFid.resize(2);
2548 0 : fidArray.buffers =
2549 0 : const_cast<const void **>(apBuffersFid.data());
2550 0 : fids.reserve(static_cast<size_t>(array->length));
2551 0 : for (size_t iRow = 0;
2552 0 : iRow < static_cast<size_t>(array->length); ++iRow)
2553 0 : fids.push_back(m_nFeatureCount + iRow);
2554 0 : fidArray.buffers[1] = fids.data();
2555 0 : fidArray.length = array->length;
2556 0 : fidSchema.release = DummyFreeSchema;
2557 0 : fidSchema.name = m_osFIDColumn.c_str();
2558 0 : fidSchema.format = "l"; // int64
2559 0 : continue;
2560 : }
2561 : }
2562 6 : else if (nGeomFieldCount == 1 &&
2563 3 : m_poFeatureDefn->GetGeomFieldIndex(
2564 3 : m_poSchema->field(i)->name().c_str()) == 0)
2565 : {
2566 : oIter =
2567 3 : oMapSchemaChildrenNameToIdx.find(pszSingleGeomFieldName);
2568 3 : if (oIter != oMapSchemaChildrenNameToIdx.end())
2569 3 : bRebuildBatch = true;
2570 : }
2571 :
2572 4 : if (oIter == oMapSchemaChildrenNameToIdx.end())
2573 : {
2574 0 : CPLError(CE_Failure, CPLE_AppDefined,
2575 : "Cannot find field '%s' in schema",
2576 0 : m_poSchema->field(i)->name().c_str());
2577 0 : return false;
2578 : }
2579 : }
2580 1516 : anMapLayerSchemaToArraySchema[i] = oIter->second;
2581 1516 : oSetReferencedFieldsInArraySchema.insert(oIter->second);
2582 : }
2583 :
2584 : // Note: we cheat a bit by declaring a single instance of the minx/miny/
2585 : // maxx/maxy sub-field ArrowSchema*, and make all struct ArrowSchema point
2586 : // to them. That's OK because we use DummyFreeSchema to release, which does
2587 : // nothing.
2588 : struct ArrowSchema bboxStructSchemaXMin;
2589 : struct ArrowSchema bboxStructSchemaYMin;
2590 : struct ArrowSchema bboxStructSchemaXMax;
2591 : struct ArrowSchema bboxStructSchemaYMax;
2592 126 : constexpr int BBOX_SUBFIELD_COUNT = 4;
2593 : std::array<struct ArrowSchema *, BBOX_SUBFIELD_COUNT>
2594 : bboxStructSchemaChildren;
2595 126 : constexpr int BBOX_STRUCT_BUFFER_COUNT = 1; // validity bitmap array
2596 : // cppcheck-suppress constStatement
2597 : std::vector<std::array<const void *, BBOX_STRUCT_BUFFER_COUNT>>
2598 252 : bboxStructBuffersPtr;
2599 252 : std::vector<std::vector<GByte>> aabyBboxStructValidity;
2600 252 : std::vector<std::vector<float>> aadfMinX, aadfMinY, aadfMaxX, aadfMaxY;
2601 : // cppcheck-suppress constStatement
2602 252 : std::vector<std::array<struct ArrowArray, BBOX_SUBFIELD_COUNT>> bboxArrays;
2603 : // cppcheck-suppress constStatement
2604 : std::vector<std::array<struct ArrowArray *, BBOX_SUBFIELD_COUNT>>
2605 252 : bboxArraysPtr;
2606 126 : constexpr int BBOX_SUBFIELD_BUFFER_COUNT =
2607 : 2; // validity bitmap array and float array
2608 : std::vector<std::array<std::array<const void *, BBOX_SUBFIELD_BUFFER_COUNT>,
2609 : BBOX_SUBFIELD_COUNT>>
2610 252 : bboxBuffersPtr;
2611 :
2612 : // Temporary arrays to hold the geometry bounding boxes.
2613 252 : std::vector<struct ArrowArray> bboxStructArray;
2614 252 : std::vector<struct ArrowSchema> bboxStructSchema;
2615 :
2616 252 : std::vector<struct ArrowSchema *> newSchemaChildren;
2617 252 : std::vector<struct ArrowArray *> newArrayChildren;
2618 126 : newSchemaChildren.reserve(m_poSchema->num_fields());
2619 126 : newArrayChildren.reserve(m_poSchema->num_fields());
2620 1642 : for (int i = 0; i < m_poSchema->num_fields() - nGeomFieldCountBBoxFields;
2621 : ++i)
2622 : {
2623 1516 : if (anMapLayerSchemaToArraySchema[i] < 0)
2624 : {
2625 0 : CPLAssert(m_poSchema->field(i)->name() == m_osFIDColumn);
2626 0 : newSchemaChildren.emplace_back(&fidSchema);
2627 0 : newArrayChildren.emplace_back(&fidArray);
2628 : }
2629 : else
2630 : {
2631 : newSchemaChildren.emplace_back(
2632 1516 : schema->children[anMapLayerSchemaToArraySchema[i]]);
2633 : newArrayChildren.emplace_back(
2634 1516 : array->children[anMapLayerSchemaToArraySchema[i]]);
2635 : }
2636 : }
2637 :
2638 126 : if (m_bWriteBBoxStruct)
2639 : {
2640 14 : memset(&bboxStructSchemaXMin, 0, sizeof(bboxStructSchemaXMin));
2641 14 : memset(&bboxStructSchemaYMin, 0, sizeof(bboxStructSchemaYMin));
2642 14 : memset(&bboxStructSchemaXMax, 0, sizeof(bboxStructSchemaXMax));
2643 14 : memset(&bboxStructSchemaYMax, 0, sizeof(bboxStructSchemaYMax));
2644 :
2645 14 : bboxStructSchemaXMin.release = DummyFreeSchema;
2646 14 : bboxStructSchemaXMin.name = "xmin";
2647 14 : bboxStructSchemaXMin.format = "f"; // float32
2648 :
2649 14 : bboxStructSchemaYMin.release = DummyFreeSchema;
2650 14 : bboxStructSchemaYMin.name = "ymin";
2651 14 : bboxStructSchemaYMin.format = "f"; // float32
2652 :
2653 14 : bboxStructSchemaXMax.release = DummyFreeSchema;
2654 14 : bboxStructSchemaXMax.name = "xmax";
2655 14 : bboxStructSchemaXMax.format = "f"; // float32
2656 :
2657 14 : bboxStructSchemaYMax.release = DummyFreeSchema;
2658 14 : bboxStructSchemaYMax.name = "ymax";
2659 14 : bboxStructSchemaYMax.format = "f"; // float32
2660 :
2661 : try
2662 : {
2663 14 : constexpr int XMIN_IDX = 0;
2664 14 : constexpr int YMIN_IDX = 1;
2665 14 : constexpr int XMAX_IDX = 2;
2666 14 : constexpr int YMAX_IDX = 3;
2667 14 : bboxStructSchemaChildren[XMIN_IDX] = &bboxStructSchemaXMin;
2668 : // cppcheck-suppress objectIndex
2669 14 : bboxStructSchemaChildren[YMIN_IDX] = &bboxStructSchemaYMin;
2670 : // cppcheck-suppress objectIndex
2671 14 : bboxStructSchemaChildren[XMAX_IDX] = &bboxStructSchemaXMax;
2672 : // cppcheck-suppress objectIndex
2673 14 : bboxStructSchemaChildren[YMAX_IDX] = &bboxStructSchemaYMax;
2674 :
2675 14 : bboxStructArray.resize(nGeomFieldCount);
2676 14 : bboxStructSchema.resize(nGeomFieldCount);
2677 14 : bboxArrays.resize(nGeomFieldCount);
2678 14 : bboxArraysPtr.resize(nGeomFieldCount);
2679 14 : bboxBuffersPtr.resize(nGeomFieldCount);
2680 14 : bboxStructBuffersPtr.resize(nGeomFieldCount);
2681 14 : aabyBboxStructValidity.resize(nGeomFieldCount);
2682 28 : memset(bboxStructArray.data(), 0,
2683 14 : nGeomFieldCount * sizeof(bboxStructArray[0]));
2684 28 : memset(bboxStructSchema.data(), 0,
2685 14 : nGeomFieldCount * sizeof(bboxStructSchema[0]));
2686 28 : memset(bboxArrays.data(), 0,
2687 14 : nGeomFieldCount * sizeof(bboxArrays[0]));
2688 14 : aadfMinX.resize(nGeomFieldCount);
2689 14 : aadfMinY.resize(nGeomFieldCount);
2690 14 : aadfMaxX.resize(nGeomFieldCount);
2691 14 : aadfMaxY.resize(nGeomFieldCount);
2692 28 : for (int i = 0; i < nGeomFieldCount; ++i)
2693 : {
2694 14 : const bool bIsNullable = CPL_TO_BOOL(
2695 14 : m_poFeatureDefn->GetGeomFieldDefn(i)->IsNullable());
2696 14 : aadfMinX[i].reserve(static_cast<size_t>(array->length));
2697 14 : aadfMinY[i].reserve(static_cast<size_t>(array->length));
2698 14 : aadfMaxX[i].reserve(static_cast<size_t>(array->length));
2699 14 : aadfMaxY[i].reserve(static_cast<size_t>(array->length));
2700 14 : aabyBboxStructValidity[i].resize(
2701 14 : static_cast<size_t>(array->length + 7) / 8, 0xFF);
2702 :
2703 14 : bboxStructSchema[i].release = DummyFreeSchema;
2704 14 : bboxStructSchema[i].name = m_apoFieldsBBOX[i]->name().c_str();
2705 14 : bboxStructSchema[i].format = "+s"; // structure
2706 14 : bboxStructSchema[i].flags =
2707 14 : bIsNullable ? ARROW_FLAG_NULLABLE : 0;
2708 14 : bboxStructSchema[i].n_children = BBOX_SUBFIELD_COUNT;
2709 14 : bboxStructSchema[i].children = bboxStructSchemaChildren.data();
2710 :
2711 14 : constexpr int VALIDITY_ARRAY_IDX = 0;
2712 14 : constexpr int BBOX_SUBFIELD_FLOAT_VALUE_IDX = 1;
2713 14 : bboxBuffersPtr[i][XMIN_IDX][BBOX_SUBFIELD_FLOAT_VALUE_IDX] =
2714 14 : aadfMinX[i].data();
2715 14 : bboxBuffersPtr[i][YMIN_IDX][BBOX_SUBFIELD_FLOAT_VALUE_IDX] =
2716 14 : aadfMinY[i].data();
2717 14 : bboxBuffersPtr[i][XMAX_IDX][BBOX_SUBFIELD_FLOAT_VALUE_IDX] =
2718 14 : aadfMaxX[i].data();
2719 14 : bboxBuffersPtr[i][YMAX_IDX][BBOX_SUBFIELD_FLOAT_VALUE_IDX] =
2720 14 : aadfMaxY[i].data();
2721 :
2722 70 : for (int j = 0; j < BBOX_SUBFIELD_COUNT; ++j)
2723 : {
2724 56 : bboxBuffersPtr[i][j][VALIDITY_ARRAY_IDX] = nullptr;
2725 :
2726 56 : bboxArrays[i][j].release = DummyFreeArray;
2727 56 : bboxArrays[i][j].length = array->length;
2728 56 : bboxArrays[i][j].n_buffers = BBOX_SUBFIELD_BUFFER_COUNT;
2729 56 : bboxArrays[i][j].buffers = bboxBuffersPtr[i][j].data();
2730 :
2731 56 : bboxArraysPtr[i][j] = &bboxArrays[i][j];
2732 : }
2733 :
2734 14 : bboxStructArray[i].release = DummyFreeArray;
2735 14 : bboxStructArray[i].n_children = BBOX_SUBFIELD_COUNT;
2736 : // coverity[escape]
2737 14 : bboxStructArray[i].children = bboxArraysPtr[i].data();
2738 14 : bboxStructArray[i].length = array->length;
2739 14 : bboxStructArray[i].n_buffers = BBOX_STRUCT_BUFFER_COUNT;
2740 14 : bboxStructBuffersPtr[i][VALIDITY_ARRAY_IDX] =
2741 14 : bIsNullable ? aabyBboxStructValidity[i].data() : nullptr;
2742 : // coverity[escape]
2743 14 : bboxStructArray[i].buffers = bboxStructBuffersPtr[i].data();
2744 :
2745 14 : newSchemaChildren.emplace_back(&bboxStructSchema[i]);
2746 14 : newArrayChildren.emplace_back(&bboxStructArray[i]);
2747 : }
2748 : }
2749 0 : catch (const std::bad_alloc &)
2750 : {
2751 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
2752 : "Out of memory in "
2753 : "OGRArrowWriterLayer::WriteArrowBatchInternal()");
2754 0 : return false;
2755 : }
2756 : }
2757 :
2758 1644 : for (int i = 0; i < static_cast<int>(schema->n_children); ++i)
2759 : {
2760 1518 : if (!cpl::contains(oSetReferencedFieldsInArraySchema, i))
2761 : {
2762 4 : if (m_osFIDColumn.empty() &&
2763 2 : strcmp(schema->children[i]->name, pszFIDName) == 0)
2764 : {
2765 : // If the input data contains a FID column, but the output data
2766 : // does not, then ignore it.
2767 : }
2768 : else
2769 : {
2770 0 : CPLError(CE_Failure, CPLE_AppDefined,
2771 : "Found field '%s' in array schema that does not exist "
2772 : "in layer schema",
2773 0 : schema->children[i]->name);
2774 0 : return false;
2775 : }
2776 : }
2777 : }
2778 :
2779 : // ImportSchema() would release the schema, but we don't want that
2780 : // So copy the structure content into a local variable, and override its
2781 : // release callback to a no-op. This may be a bit fragile, but it doesn't
2782 : // look like ImportSchema implementation tries to access the C ArrowSchema
2783 : // after it has been called.
2784 126 : struct ArrowSchema lSchema = *schema;
2785 126 : schema = &lSchema;
2786 126 : CPL_IGNORE_RET_VAL(schema);
2787 :
2788 126 : lSchema.n_children = newSchemaChildren.size();
2789 126 : lSchema.children = newSchemaChildren.data();
2790 :
2791 126 : lSchema.release = DummyFreeSchema;
2792 252 : auto poSchemaResult = arrow::ImportSchema(&lSchema);
2793 126 : CPLAssert(lSchema.release == nullptr);
2794 126 : if (!poSchemaResult.ok())
2795 : {
2796 0 : CPLError(CE_Failure, CPLE_AppDefined, "ImportSchema() failed with %s",
2797 0 : poSchemaResult.status().message().c_str());
2798 0 : return false;
2799 : }
2800 252 : auto poSchema = *poSchemaResult;
2801 :
2802 : // Hack the array to use the new children we've computed above
2803 : // but make sure the original release() callback sees the original children
2804 : struct ArrayReleaser
2805 : {
2806 : struct ArrowArray ori_array
2807 : {
2808 : };
2809 :
2810 126 : explicit ArrayReleaser(struct ArrowArray *array)
2811 126 : {
2812 126 : memcpy(&ori_array, array, sizeof(*array));
2813 126 : array->release = ArrayReleaser::release;
2814 126 : array->private_data = this;
2815 126 : }
2816 :
2817 126 : static void release(struct ArrowArray *array)
2818 : {
2819 126 : struct ArrayReleaser *releaser =
2820 : static_cast<struct ArrayReleaser *>(array->private_data);
2821 126 : memcpy(array, &(releaser->ori_array), sizeof(*array));
2822 126 : CPLAssert(array->release != nullptr);
2823 126 : array->release(array);
2824 126 : CPLAssert(array->release == nullptr);
2825 126 : delete releaser;
2826 126 : }
2827 : };
2828 :
2829 : // Must be allocated on the heap, since ArrayReleaser::release() will be
2830 : // called after this method has ended.
2831 126 : ArrayReleaser *releaser = new ArrayReleaser(array);
2832 126 : array->private_data = releaser;
2833 126 : array->n_children = newArrayChildren.size();
2834 : // cppcheck-suppress autoVariables
2835 126 : array->children = newArrayChildren.data();
2836 :
2837 : // Process geometry columns:
2838 : // - if the output encoding is WKB, then just note the geometry type and
2839 : // envelope.
2840 : // - otherwise convert to the output encoding.
2841 126 : int nBuilderIdx = 0;
2842 126 : if (!m_osFIDColumn.empty())
2843 : {
2844 2 : nBuilderIdx++;
2845 : }
2846 : std::map<std::string, std::shared_ptr<arrow::Array>>
2847 252 : oMapGeomFieldNameToArray;
2848 252 : for (int i = 0; i < nGeomFieldCount; ++i, ++nBuilderIdx)
2849 : {
2850 : const char *pszThisGeomFieldName =
2851 126 : m_poFeatureDefn->GetGeomFieldDefn(i)->GetNameRef();
2852 126 : int nIdx = poSchema->GetFieldIndex(pszThisGeomFieldName);
2853 126 : if (nIdx < 0)
2854 : {
2855 3 : if (nGeomFieldCount == 1)
2856 3 : nIdx = poSchema->GetFieldIndex(pszSingleGeomFieldName);
2857 3 : if (nIdx < 0)
2858 : {
2859 0 : CPLError(CE_Failure, CPLE_AppDefined,
2860 : "Cannot find geometry field '%s' in schema",
2861 : pszThisGeomFieldName);
2862 0 : return false;
2863 : }
2864 : }
2865 :
2866 126 : if (strcmp(lSchema.children[nIdx]->format, "z") != 0 &&
2867 1 : strcmp(lSchema.children[nIdx]->format, "Z") != 0)
2868 : {
2869 0 : CPLError(CE_Failure, CPLE_AppDefined,
2870 : "Type of geometry field '%s' is not binary, but '%s'",
2871 0 : pszThisGeomFieldName, lSchema.children[nIdx]->format);
2872 0 : return false;
2873 : }
2874 :
2875 126 : const auto psGeomArray = array->children[nIdx];
2876 126 : const uint8_t *pabyValidity =
2877 126 : psGeomArray->null_count != 0
2878 126 : ? static_cast<const uint8_t *>(psGeomArray->buffers[0])
2879 : : nullptr;
2880 126 : const bool bUseOffsets32 =
2881 126 : (strcmp(lSchema.children[nIdx]->format, "z") == 0);
2882 126 : const uint32_t *panOffsets32 =
2883 126 : static_cast<const uint32_t *>(psGeomArray->buffers[1]) +
2884 126 : psGeomArray->offset;
2885 126 : const uint64_t *panOffsets64 =
2886 126 : static_cast<const uint64_t *>(psGeomArray->buffers[1]) +
2887 126 : psGeomArray->offset;
2888 126 : GByte *pabyData =
2889 126 : static_cast<GByte *>(const_cast<void *>(psGeomArray->buffers[2]));
2890 126 : OGREnvelope sEnvelope;
2891 126 : auto poBuilder = m_apoBuilders[nBuilderIdx].get();
2892 :
2893 681 : for (size_t iRow = 0; iRow < static_cast<size_t>(psGeomArray->length);
2894 : ++iRow)
2895 : {
2896 555 : bool bValidGeom = false;
2897 :
2898 1059 : if (!pabyValidity ||
2899 504 : TestBit(pabyValidity,
2900 504 : static_cast<size_t>(iRow + psGeomArray->offset)))
2901 : {
2902 439 : const auto nLen =
2903 439 : bUseOffsets32 ? static_cast<size_t>(panOffsets32[iRow + 1] -
2904 429 : panOffsets32[iRow])
2905 10 : : static_cast<size_t>(panOffsets64[iRow + 1] -
2906 10 : panOffsets64[iRow]);
2907 439 : GByte *pabyWkb =
2908 439 : pabyData + (bUseOffsets32
2909 429 : ? panOffsets32[iRow]
2910 10 : : static_cast<size_t>(panOffsets64[iRow]));
2911 439 : if (m_aeGeomEncoding[i] == OGRArrowGeomEncoding::WKB)
2912 : {
2913 171 : FixupWKBGeometryBeforeWriting(pabyWkb, nLen);
2914 :
2915 171 : uint32_t nType = 0;
2916 171 : bool bNeedSwap = false;
2917 171 : if (OGRWKBGetGeomType(pabyWkb, nLen, bNeedSwap, nType))
2918 : {
2919 171 : m_oSetWrittenGeometryTypes[i].insert(
2920 171 : static_cast<OGRwkbGeometryType>(nType));
2921 171 : if (OGRWKBGetBoundingBox(pabyWkb, nLen, sEnvelope))
2922 : {
2923 171 : bValidGeom = true;
2924 171 : m_aoEnvelopes[i].Merge(sEnvelope);
2925 :
2926 171 : if (m_bWriteBBoxStruct)
2927 : {
2928 43 : aadfMinX[i].push_back(
2929 43 : castToFloatDown(sEnvelope.MinX));
2930 43 : aadfMinY[i].push_back(
2931 43 : castToFloatDown(sEnvelope.MinY));
2932 43 : aadfMaxX[i].push_back(
2933 43 : castToFloatUp(sEnvelope.MaxX));
2934 43 : aadfMaxY[i].push_back(
2935 43 : castToFloatUp(sEnvelope.MaxY));
2936 : }
2937 : }
2938 : }
2939 : }
2940 : else
2941 : {
2942 268 : size_t nBytesConsumedOut = 0;
2943 268 : OGRGeometry *poGeometry = nullptr;
2944 268 : OGRGeometryFactory::createFromWkb(
2945 : pabyWkb, nullptr, &poGeometry, nLen, wkbVariantIso,
2946 : nBytesConsumedOut);
2947 268 : if (BuildGeometry(poGeometry, i, poBuilder) != OGRERR_NONE)
2948 : {
2949 0 : delete poGeometry;
2950 0 : return false;
2951 : }
2952 268 : bValidGeom = true;
2953 268 : if (m_bWriteBBoxStruct)
2954 : {
2955 0 : poGeometry->getEnvelope(&sEnvelope);
2956 0 : aadfMinX[i].push_back(castToFloatDown(sEnvelope.MinX));
2957 0 : aadfMinY[i].push_back(castToFloatDown(sEnvelope.MinY));
2958 0 : aadfMaxX[i].push_back(castToFloatUp(sEnvelope.MaxX));
2959 0 : aadfMaxY[i].push_back(castToFloatUp(sEnvelope.MaxY));
2960 : }
2961 268 : delete poGeometry;
2962 : }
2963 : }
2964 : else
2965 : {
2966 116 : if (m_aeGeomEncoding[i] != OGRArrowGeomEncoding::WKB)
2967 : {
2968 81 : if (BuildGeometry(nullptr, i, poBuilder) != OGRERR_NONE)
2969 0 : return false;
2970 : }
2971 : }
2972 :
2973 555 : if (!bValidGeom && m_bWriteBBoxStruct)
2974 : {
2975 6 : if ((bboxStructSchema[i].flags & ARROW_FLAG_NULLABLE))
2976 : {
2977 6 : bboxStructArray[i].null_count++;
2978 6 : aabyBboxStructValidity[i][iRow / 8] &=
2979 6 : ~(1 << static_cast<int>(iRow % 8));
2980 : }
2981 6 : aadfMinX[i].push_back(0.0f);
2982 6 : aadfMinY[i].push_back(0.0f);
2983 6 : aadfMaxX[i].push_back(0.0f);
2984 6 : aadfMaxY[i].push_back(0.0f);
2985 : }
2986 : }
2987 :
2988 126 : if (m_aeGeomEncoding[i] != OGRArrowGeomEncoding::WKB)
2989 : {
2990 0 : std::shared_ptr<arrow::Array> geomArray;
2991 81 : auto status = poBuilder->Finish(&geomArray);
2992 81 : if (!status.ok())
2993 : {
2994 0 : CPLError(CE_Failure, CPLE_AppDefined,
2995 : "builder::Finish() for field %s failed with %s",
2996 0 : pszThisGeomFieldName, status.message().c_str());
2997 0 : return false;
2998 : }
2999 162 : oMapGeomFieldNameToArray[pszThisGeomFieldName] =
3000 162 : std::move(geomArray);
3001 : }
3002 : }
3003 :
3004 : auto poRecordBatchResult =
3005 252 : arrow::ImportRecordBatch(array, std::move(poSchema));
3006 126 : if (!poRecordBatchResult.ok())
3007 : {
3008 0 : CPLError(CE_Failure, CPLE_AppDefined,
3009 : "ImportRecordBatch() failed with %s",
3010 0 : poRecordBatchResult.status().message().c_str());
3011 0 : return false;
3012 : }
3013 252 : auto poRecordBatch = *poRecordBatchResult;
3014 :
3015 126 : if (!(bRebuildBatch || !oMapGeomFieldNameToArray.empty()))
3016 : {
3017 994 : for (int i = 0; i < m_poSchema->num_fields(); ++i)
3018 : {
3019 : const auto oIter =
3020 952 : oMapGeomFieldNameToArray.find(m_poSchema->field(i)->name());
3021 952 : auto l_array = (oIter != oMapGeomFieldNameToArray.end())
3022 0 : ? oIter->second
3023 1904 : : poRecordBatch->column(i);
3024 1904 : const auto schemaType = m_poSchema->field(i)->type();
3025 1904 : const auto arrayType = l_array->type();
3026 1904 : if (schemaType->id() != arrow::Type::EXTENSION &&
3027 952 : arrayType->id() == arrow::Type::EXTENSION)
3028 : {
3029 0 : bRebuildBatch = true;
3030 : }
3031 952 : else if (schemaType->id() != arrayType->id())
3032 : {
3033 3 : CPLDebug(
3034 : "Arrow",
3035 : "Field idx=%d name='%s', schema type=%s, array type=%s", i,
3036 1 : m_poSchema->field(i)->name().c_str(),
3037 2 : schemaType->ToString().c_str(),
3038 2 : arrayType->ToString().c_str());
3039 : }
3040 : }
3041 : }
3042 :
3043 : // below assertion commented out since it is not strictly necessary, but
3044 : // reflects what ImportRecordBatch() does.
3045 : // CPLAssert(array->release == nullptr);
3046 :
3047 : // We may need to reconstruct a final record batch that perfectly matches
3048 : // the expected schema.
3049 126 : if (bRebuildBatch || !oMapGeomFieldNameToArray.empty())
3050 : {
3051 84 : std::vector<std::shared_ptr<arrow::Array>> apoArrays;
3052 662 : for (int i = 0; i < m_poSchema->num_fields(); ++i)
3053 : {
3054 : const auto oIter =
3055 578 : oMapGeomFieldNameToArray.find(m_poSchema->field(i)->name());
3056 578 : if (oIter != oMapGeomFieldNameToArray.end())
3057 81 : apoArrays.emplace_back(oIter->second);
3058 : else
3059 497 : apoArrays.emplace_back(poRecordBatch->column(i));
3060 :
3061 578 : auto expectedFieldType = m_poSchema->field(i)->type();
3062 578 : if (expectedFieldType->id() == arrow::Type::EXTENSION)
3063 : {
3064 0 : auto extensionType = cpl::down_cast<arrow::ExtensionType *>(
3065 : expectedFieldType.get());
3066 0 : expectedFieldType = extensionType->storage_type();
3067 : }
3068 :
3069 578 : if (apoArrays.back()->type()->id() == arrow::Type::EXTENSION)
3070 : {
3071 0 : apoArrays.back() =
3072 0 : std::static_pointer_cast<arrow::ExtensionArray>(
3073 0 : apoArrays.back())
3074 0 : ->storage();
3075 : }
3076 :
3077 578 : if (apoArrays.back()->type()->id() != expectedFieldType->id())
3078 : {
3079 0 : CPLError(
3080 : CE_Failure, CPLE_AppDefined,
3081 : "Field '%s' of unexpected type. Got '%s', expected '%s'",
3082 0 : m_poSchema->field(i)->name().c_str(),
3083 0 : apoArrays.back()->type()->name().c_str(),
3084 0 : expectedFieldType->name().c_str());
3085 0 : return false;
3086 : }
3087 : }
3088 336 : poRecordBatchResult = arrow::RecordBatch::Make(
3089 252 : m_poSchema, poRecordBatch->num_rows(), std::move(apoArrays));
3090 84 : if (!poRecordBatchResult.ok())
3091 : {
3092 0 : CPLError(CE_Failure, CPLE_AppDefined,
3093 : "RecordBatch::Make() failed with %s",
3094 0 : poRecordBatchResult.status().message().c_str());
3095 0 : return false;
3096 : }
3097 84 : poRecordBatch = *poRecordBatchResult;
3098 : }
3099 :
3100 126 : if (writeBatch(poRecordBatch))
3101 : {
3102 126 : m_nFeatureCount += poRecordBatch->num_rows();
3103 126 : return true;
3104 : }
3105 0 : return false;
3106 : #endif
3107 : }
3108 :
3109 : #endif /* OGARROWWRITERLAYER_HPP_INCLUDED */
|
