Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: OpenGIS Simple Features Reference Implementation
4 : * Purpose: Implementation of OGC Features and Geometries JSON (JSON-FG)
5 : * Author: Even Rouault <even.rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2023, Even Rouault <even.rouault at spatialys.com>
9 : *
10 : * SPDX-License-Identifier: MIT
11 : ****************************************************************************/
12 :
13 : #ifndef OGR_JSONFG_H_INCLUDED
14 : #define OGR_JSONFG_H_INCLUDED
15 :
16 : #include "cpl_vsi_virtual.h"
17 :
18 : #include "gdal_priv.h"
19 : #include "ogrsf_frmts.h"
20 : #include "ogrgeojsonutils.h"
21 : #include "ogrgeojsonwriter.h"
22 : #include "ogrjsoncollectionstreamingparser.h"
23 : #include "memdataset.h"
24 : #include "directedacyclicgraph.hpp"
25 :
26 : #include <map>
27 : #include <set>
28 : #include <utility>
29 :
30 : /************************************************************************/
31 : /* OGRJSONFGMemLayer */
32 : /************************************************************************/
33 :
34 : /** Layer with all features ingested into memory. */
35 152 : class OGRJSONFGMemLayer final : public OGRMemLayer
36 : {
37 : public:
38 : OGRJSONFGMemLayer(GDALDataset *poDS, const char *pszName,
39 : OGRSpatialReference *poSRS, OGRwkbGeometryType eGType);
40 : ~OGRJSONFGMemLayer() override;
41 :
42 3 : const char *GetFIDColumn() const override
43 : {
44 3 : return osFIDColumn_.c_str();
45 : }
46 :
47 0 : void SetFIDColumn(const char *pszName)
48 : {
49 0 : osFIDColumn_ = pszName;
50 0 : }
51 :
52 : void AddFeature(std::unique_ptr<OGRFeature> poFeature);
53 :
54 23 : GDALDataset *GetDataset() override
55 : {
56 23 : return m_poDS;
57 : }
58 :
59 : private:
60 : GDALDataset *m_poDS = nullptr;
61 : std::string osFIDColumn_{};
62 : bool bOriginalIdModified_ = false;
63 :
64 : CPL_DISALLOW_COPY_ASSIGN(OGRJSONFGMemLayer)
65 : };
66 :
67 : /************************************************************************/
68 : /* OGRJSONFGStreamedLayer */
69 : /************************************************************************/
70 :
71 : class OGRJSONFGStreamingParser;
72 :
73 : /** Layer with features being acquired progressively through a streaming
74 : parser.
75 :
76 : Only applies for FeatureCollection read through a file
77 : */
78 : class OGRJSONFGStreamedLayer final
79 : : public OGRLayer,
80 : public OGRGetNextFeatureThroughRaw<OGRJSONFGStreamedLayer>
81 : {
82 : public:
83 : OGRJSONFGStreamedLayer(GDALDataset *poDS, const char *pszName,
84 : OGRSpatialReference *poSRS,
85 : OGRwkbGeometryType eGType);
86 : ~OGRJSONFGStreamedLayer() override;
87 :
88 : // BEGIN specific public API
89 :
90 : //! Set the FID column name
91 0 : void SetFIDColumn(const char *pszName)
92 : {
93 0 : osFIDColumn_ = pszName;
94 0 : }
95 :
96 : //! Set the total feature count
97 130 : void SetFeatureCount(GIntBig nCount)
98 : {
99 130 : nFeatureCount_ = nCount;
100 130 : }
101 :
102 : /** Set the file handle.
103 :
104 : Must be called before GetNextFeature() is called
105 : */
106 : void SetFile(VSIVirtualHandleUniquePtr &&poFile);
107 :
108 : /** Set the streaming parser
109 :
110 : Must be called before GetNextFeature() is called
111 : */
112 : void SetStreamingParser(
113 : std::unique_ptr<OGRJSONFGStreamingParser> &&poStreamingParser);
114 :
115 : // END specific public API
116 :
117 546 : const char *GetFIDColumn() const override
118 : {
119 546 : return osFIDColumn_.c_str();
120 : }
121 :
122 1875 : const OGRFeatureDefn *GetLayerDefn() const override
123 : {
124 1875 : return poFeatureDefn_;
125 : }
126 :
127 : int TestCapability(const char *pszCap) const override;
128 :
129 : GIntBig GetFeatureCount(int bForce) override;
130 :
131 : void ResetReading() override;
132 :
133 358 : DEFINE_GET_NEXT_FEATURE_THROUGH_RAW(OGRJSONFGStreamedLayer)
134 :
135 16 : GDALDataset *GetDataset() override
136 : {
137 16 : return m_poDS;
138 : }
139 :
140 : private:
141 : GDALDataset *m_poDS = nullptr;
142 : OGRFeatureDefn *poFeatureDefn_ = nullptr;
143 : std::string osFIDColumn_{};
144 :
145 : /** Total number of features. */
146 : GIntBig nFeatureCount_ = -1;
147 :
148 : VSIVirtualHandleUniquePtr poFile_{};
149 :
150 : std::unique_ptr<OGRJSONFGStreamingParser> poStreamingParser_{};
151 :
152 : /** Whether a warning has been emitted about feature IDs having been
153 : * modified */
154 : bool bOriginalIdModified_ = false;
155 : /** Set of feature IDs read/allocated up to that point */
156 : std::set<GIntBig> oSetUsedFIDs_{};
157 :
158 : /** Ensure the FID of the feature is unique */
159 : OGRFeature *EnsureUniqueFID(OGRFeature *poFeat);
160 :
161 : /** Return next feature (without filter) */
162 : OGRFeature *GetNextRawFeature();
163 :
164 : CPL_DISALLOW_COPY_ASSIGN(OGRJSONFGStreamedLayer)
165 : };
166 :
167 : /************************************************************************/
168 : /* OGRJSONFGWriteLayer */
169 : /************************************************************************/
170 :
171 : class OGRJSONFGDataset;
172 :
173 : class OGRJSONFGWriteLayer final : public OGRLayer
174 : {
175 : public:
176 : OGRJSONFGWriteLayer(
177 : const char *pszName, const OGRSpatialReference *poSRS,
178 : std::unique_ptr<OGRCoordinateTransformation> &&poCTToWGS84,
179 : const std::string &osCoordRefSys, OGRwkbGeometryType eGType,
180 : CSLConstList papszOptions, OGRJSONFGDataset *poDS);
181 : ~OGRJSONFGWriteLayer() override;
182 :
183 : //
184 : // OGRLayer Interface
185 : //
186 682 : const OGRFeatureDefn *GetLayerDefn() const override
187 : {
188 682 : return poFeatureDefn_;
189 : }
190 :
191 1 : const OGRSpatialReference *GetSpatialRef() const override
192 : {
193 1 : return nullptr;
194 : }
195 :
196 16 : void ResetReading() override
197 : {
198 16 : }
199 :
200 16 : OGRFeature *GetNextFeature() override
201 : {
202 16 : return nullptr;
203 : }
204 :
205 : OGRErr ICreateFeature(OGRFeature *poFeature) override;
206 : OGRErr CreateField(const OGRFieldDefn *poField, int bApproxOK) override;
207 : int TestCapability(const char *pszCap) const override;
208 :
209 : OGRErr SyncToDisk() override;
210 :
211 : GDALDataset *GetDataset() override;
212 :
213 125 : bool HasPolyhedra() const
214 : {
215 125 : return m_bPolyhedraWritten;
216 : }
217 :
218 125 : bool HasCurve() const
219 : {
220 125 : return m_bCurveWritten;
221 : }
222 :
223 125 : bool HasMeasure() const
224 : {
225 125 : return m_bMeasureWritten;
226 : }
227 :
228 : private:
229 : OGRJSONFGDataset *poDS_{};
230 : OGRFeatureDefn *poFeatureDefn_ = nullptr;
231 : std::unique_ptr<OGRCoordinateTransformation> poCTToWGS84_;
232 : bool bIsWGS84CRS_ = false;
233 : bool m_bMustSwapForPlace = false;
234 : int nOutCounter_ = 0;
235 : std::string osCoordRefSys_{};
236 : bool m_bPolyhedraWritten = false;
237 : bool m_bCurveWritten = false;
238 : bool m_bMeasureWritten = false;
239 : bool bLayerLevelMeasuresWritten_ = false;
240 : std::string osMeasureUnit_{};
241 : std::string osMeasureDescription_{};
242 :
243 : OGRGeoJSONWriteOptions oWriteOptions_{};
244 : OGRGeoJSONWriteOptions oWriteOptionsPlace_{};
245 : bool bWriteFallbackGeometry_ = true;
246 :
247 : CPL_DISALLOW_COPY_ASSIGN(OGRJSONFGWriteLayer)
248 : };
249 :
250 : /************************************************************************/
251 : /* OGRJSONFGDataset */
252 : /************************************************************************/
253 :
254 : class OGRJSONFGReader;
255 :
256 : class OGRJSONFGDataset final : public GDALDataset
257 : {
258 : public:
259 317 : OGRJSONFGDataset() = default;
260 : ~OGRJSONFGDataset() override;
261 :
262 : CPLErr Close() override;
263 :
264 : bool Open(GDALOpenInfo *poOpenInfo, GeoJSONSourceType nSrcType);
265 : bool Create(const char *pszName, CSLConstList papszOptions);
266 :
267 58 : int GetLayerCount() const override
268 : {
269 58 : return static_cast<int>(apoLayers_.size());
270 : }
271 :
272 : const OGRLayer *GetLayer(int i) const override;
273 :
274 : //! Return the output file handle. Used by OGRJSONFGWriteLayer
275 296 : VSILFILE *GetOutputFile() const
276 : {
277 296 : return fpOut_;
278 : }
279 :
280 : /** Return whether there is a single output layer.
281 : * Used by OGRJSONFGWriteLayer
282 : */
283 320 : bool IsSingleOutputLayer() const
284 : {
285 320 : return bSingleOutputLayer_;
286 : }
287 :
288 : //! Return whether the output file is seekable
289 5 : bool GetFpOutputIsSeekable() const
290 : {
291 5 : return bFpOutputIsSeekable_;
292 : }
293 :
294 : void BeforeCreateFeature();
295 :
296 : OGRLayer *ICreateLayer(const char *pszName,
297 : const OGRGeomFieldDefn *poGeomFieldDefn,
298 : CSLConstList papszOptions) override;
299 :
300 : int TestCapability(const char *pszCap) const override;
301 :
302 : OGRErr SyncToDiskInternal();
303 :
304 : protected:
305 : friend class OGRJSONFGReader;
306 : OGRJSONFGMemLayer *AddLayer(std::unique_ptr<OGRJSONFGMemLayer> &&poLayer);
307 : OGRJSONFGStreamedLayer *
308 : AddLayer(std::unique_ptr<OGRJSONFGStreamedLayer> &&poLayer);
309 :
310 : private:
311 : char *pszGeoData_ = nullptr;
312 : size_t nGeoDataLen_ = 0;
313 : std::vector<std::unique_ptr<OGRLayer>> apoLayers_{};
314 : std::unique_ptr<OGRJSONFGReader> poReader_{};
315 :
316 : // Write side
317 : VSILFILE *fpOut_ = nullptr;
318 : vsi_l_offset m_nPositionBeforeConformsTo = 0;
319 : vsi_l_offset m_nPositionAfterConformsTo = 0;
320 : bool bSingleOutputLayer_ = false;
321 : bool bHasEmittedFeatures_ = false;
322 : bool bFpOutputIsSeekable_ = false;
323 :
324 : /** Offset at which the '] }' terminating sequence has already been
325 : * written by SyncToDisk(). 0 if it has not been written.
326 : */
327 : vsi_l_offset m_nPositionBeforeFCClosed = 0;
328 :
329 : bool ReadFromFile(GDALOpenInfo *poOpenInfo, const char *pszUnprefixed);
330 : bool ReadFromService(GDALOpenInfo *poOpenInfo, const char *pszSource);
331 :
332 : bool FinishWriting();
333 :
334 : bool EmitStartFeaturesIfNeededAndReturnIfFirstFeature();
335 :
336 : CPL_DISALLOW_COPY_ASSIGN(OGRJSONFGDataset)
337 : };
338 :
339 : /************************************************************************/
340 : /* OGRJSONFGReader */
341 : /************************************************************************/
342 :
343 : class OGRJSONFGReader
344 : {
345 : public:
346 232 : OGRJSONFGReader() = default;
347 : ~OGRJSONFGReader();
348 :
349 : /** Load all features from the passed in JSON text in OGRJSONFGMemLayer(s)
350 : *
351 : * This method should only be called once, and is exclusive with
352 : * AnalyzeWithStreamingParser()
353 : */
354 : bool Load(OGRJSONFGDataset *poDS, const char *pszText,
355 : const std::string &osDefaultLayerName);
356 :
357 : /** Do a first pass analysis of the content of the passed file to create
358 : * OGRJSONFGStreamedLayer's
359 : *
360 : * It is the responsibility of the caller to call
361 : * SetFile() and SetStreamingParser() on the created layers afterwards
362 : *
363 : * This method should only be called once, and is exclusive with
364 : * Load()
365 : */
366 : bool AnalyzeWithStreamingParser(OGRJSONFGDataset *poDS, VSILFILE *fp,
367 : const std::string &osDefaultLayerName,
368 : bool &bCanTryWithNonStreamingParserOut,
369 : bool &bHasTopLevelMeasures);
370 :
371 : /** Geometry element we are interested in. */
372 : enum class GeometryElement
373 : {
374 : /** Use "place" when possible, fallback to "geometry" otherwise. */
375 : AUTO,
376 : /** Only use "place" */
377 : PLACE,
378 : /** Only use "geometry" */
379 : GEOMETRY,
380 : };
381 :
382 : /** Sets the geometry element we are interested in. */
383 6 : void SetGeometryElement(GeometryElement elt)
384 : {
385 6 : eGeometryElement_ = elt;
386 6 : }
387 :
388 : /** Returns a OGRFeature built from the passed in JSON object.
389 : *
390 : * @param poObj JSON feature
391 : * @param pszRequestedLayer name of the layer of interest, or nullptr if
392 : * no filtering needed on the layer name. If the feature does not belong
393 : * to the requested layer, nullptr is returned.
394 : * @param bHasM Whether the upper level of this object has measures
395 : * @param pOutMemLayer Pointer to the OGRJSONFGMemLayer* layer to which
396 : * the returned feature belongs to. May be nullptr. Only applies when
397 : * the Load() method has been used.
398 : * @param pOutStreamedLayer Pointer to the OGRJSONFGStreamedLayer* layer to
399 : * which the returned feature belongs to. May be nullptr. Only applies when
400 : * the AnalyzeWithStreamingParser() method has been used.
401 : */
402 : std::unique_ptr<OGRFeature>
403 : ReadFeature(json_object *poObj, const char *pszRequestedLayer, bool bHasM,
404 : OGRJSONFGMemLayer **pOutMemLayer,
405 : OGRJSONFGStreamedLayer **pOutStreamedLayer);
406 :
407 : protected:
408 : friend class OGRJSONFGStreamingParser;
409 :
410 : bool GenerateLayerDefnFromFeature(json_object *poObj);
411 :
412 : private:
413 : GeometryElement eGeometryElement_ = GeometryElement::AUTO;
414 :
415 : OGRJSONFGDataset *poDS_ = nullptr;
416 : std::string osDefaultLayerName_{};
417 : json_object *poObject_ = nullptr;
418 :
419 : bool bFlattenNestedAttributes_ = false;
420 : char chNestedAttributeSeparator_ = 0;
421 : bool bArrayAsString_ = false;
422 : bool bDateAsString_ = false;
423 : std::string osMeasureUnit_{};
424 : std::string osMeasureDescription_{};
425 :
426 : /** Layer building context, specific to one layer. */
427 225 : struct LayerDefnBuildContext
428 : {
429 : //! Maps a field name to its index in apoFieldDefn[]
430 : std::map<std::string, int> oMapFieldNameToIdx{};
431 :
432 : //! Vector of OGRFieldDefn
433 : std::vector<std::unique_ptr<OGRFieldDefn>> apoFieldDefn{};
434 :
435 : //! Directed acyclic graph used to build the order of fields.
436 : gdal::DirectedAcyclicGraph<int, std::string> dag{};
437 :
438 : /** Set of indices of apoFieldDefn[] for which no type information is
439 : * known yet. */
440 : std::set<int> aoSetUndeterminedTypeFields{};
441 :
442 : //! Whether at least one feature has a "coordRefSys" member.
443 : bool bHasCoordRefSysAtFeatureLevel = false;
444 :
445 : /** CRS object corresponding to "coordRefsys" member at feature level.
446 : * Only set if homogeneous among features.
447 : */
448 : std::unique_ptr<OGRSpatialReference> poCRSAtFeatureLevel{};
449 :
450 : /** Serialized JSON value of "coordRefsys" member at feature level.
451 : * Only set if homogeneous among features.
452 : */
453 : std::string osCoordRefSysAtFeatureLevel{};
454 :
455 : /** Whether to switch X/Y ordinates in geometries appearing in "place"
456 : * element. Only applies to CRS at layer level.
457 : */
458 : bool bSwapPlacesXY = false;
459 :
460 : //! Whether the layer CRS is WGS 84.
461 : bool bLayerCRSIsWGS84 = false;
462 :
463 : //! Coordinate transformation from WGS 84 to layer CRS (might be null)
464 : std::unique_ptr<OGRCoordinateTransformation> poCTWGS84ToLayerCRS{};
465 :
466 : /** Feature count */
467 : GIntBig nFeatureCount = 0;
468 :
469 : //! Whether the Feature.id should be mapped to a OGR field.
470 : bool bFeatureLevelIdAsAttribute = false;
471 :
472 : //! Whether the Feature.id should be mapped to a OGR FID.
473 : bool bFeatureLevelIdAsFID = false;
474 :
475 : //! Whether 64-bit integers are needed for OGR FID.
476 : bool bNeedFID64 = false;
477 :
478 : //! Whether detection of layer geometry type is still needed.
479 : bool bDetectLayerGeomType = true;
480 :
481 : //! Whether no geometry has been analyzed yet.
482 : bool bFirstGeometry = true;
483 :
484 : //! Layer geometry type.
485 : OGRwkbGeometryType eLayerGeomType = wkbUnknown;
486 :
487 : //! Whether a Feature.time.date element has been found.
488 : bool bHasTimeDate = false;
489 :
490 : //! Whether a Feature.time.timestamp element has been found.
491 : bool bHasTimeTimestamp = false;
492 :
493 : /** Whether a Feature.time.interval[0] element of type timestamp has
494 : * been found */
495 : bool bHasTimeIntervalStartTimestamp = false;
496 :
497 : /** Whether a Feature.time.interval[0] element of type date has
498 : * been found */
499 : bool bHasTimeIntervalStartDate = false;
500 :
501 : /** Whether a Feature.time.interval[1] element of type timestamp has
502 : * been found */
503 : bool bHasTimeIntervalEndTimestamp = false;
504 :
505 : /** Whether a Feature.time.interval[1] element of type date has
506 : * been found */
507 : bool bHasTimeIntervalEndDate = false;
508 :
509 : //! Index of OGR field "time" / "jsonfg_time"
510 : int nIdxFieldTime = -1;
511 :
512 : //! Index of OGR field "time_start" / "jsonfg_time_start"
513 : int nIdxFieldTimeStart = -1;
514 :
515 : //! Index of OGR field "time_end" / "jsonfg_time_end"
516 : int nIdxFieldTimeEnd = -1;
517 :
518 : //! Corresponding OGRJSONFGMemLayer (only for Load() ingestion mode)
519 : OGRJSONFGMemLayer *poMemLayer = nullptr;
520 :
521 : /** Corresponding OGRJSONFGStreamedLayer(only for
522 : * AnalyzeWithStreamingParser() mode) */
523 : OGRJSONFGStreamedLayer *poStreamedLayer = nullptr;
524 :
525 : bool bSameMeasureMetadata = true;
526 :
527 : //! Measure unit
528 : std::string osMeasureUnit{};
529 :
530 : //! Measure description
531 : std::string osMeasureDescription{};
532 :
533 431 : LayerDefnBuildContext() = default;
534 19 : LayerDefnBuildContext(LayerDefnBuildContext &&) = default;
535 : LayerDefnBuildContext &operator=(LayerDefnBuildContext &&) = default;
536 :
537 : private:
538 : CPL_DISALLOW_COPY_ASSIGN(LayerDefnBuildContext)
539 : };
540 :
541 : //! Maps a layer name to its build context
542 : std::map<std::string, LayerDefnBuildContext> oMapBuildContext_{};
543 :
544 : //
545 : // Copy operations not supported.
546 : //
547 : CPL_DISALLOW_COPY_ASSIGN(OGRJSONFGReader)
548 :
549 : const char *GetLayerNameForFeature(json_object *poObj) const;
550 : bool GenerateLayerDefns();
551 : bool FinalizeGenerateLayerDefns(bool bStreamedLayer);
552 : void FinalizeBuildContext(LayerDefnBuildContext &oBuildContext,
553 : const char *pszLayerName, bool bStreamedLayer,
554 : bool bInvalidCRS, bool bSwapPlacesXYTopLevel,
555 : OGRSpatialReference *poSRSTopLevel);
556 : };
557 :
558 : /************************************************************************/
559 : /* OGRJSONFGStreamingParser */
560 : /************************************************************************/
561 :
562 : /** FeatureCollection streaming parser. */
563 600 : class OGRJSONFGStreamingParser final : public OGRJSONCollectionStreamingParser
564 : {
565 : OGRJSONFGReader &m_oReader;
566 : std::string m_osRequestedLayer{};
567 :
568 : std::vector<std::pair<std::unique_ptr<OGRFeature>, OGRLayer *>>
569 : m_apoFeatures{};
570 : size_t m_nCurFeatureIdx = 0;
571 :
572 : CPL_DISALLOW_COPY_ASSIGN(OGRJSONFGStreamingParser)
573 :
574 : protected:
575 : void GotFeature(json_object *poObj, bool bFirstPass,
576 : const std::string &osJson) override;
577 : void TooComplex() override;
578 :
579 : public:
580 : OGRJSONFGStreamingParser(OGRJSONFGReader &oReader, bool bFirstPass,
581 : bool bHasTopLevelMeasures);
582 : ~OGRJSONFGStreamingParser() override;
583 :
584 130 : void SetRequestedLayer(const char *pszRequestedLayer)
585 : {
586 130 : m_osRequestedLayer = pszRequestedLayer;
587 130 : }
588 :
589 : std::unique_ptr<OGRJSONFGStreamingParser> Clone();
590 :
591 : std::pair<std::unique_ptr<OGRFeature>, OGRLayer *> GetNextFeature();
592 : };
593 :
594 : bool OGRJSONFGMustSwapXY(const OGRSpatialReference *poSRS);
595 :
596 : #endif // OGR_JSONFG_H_INCLUDED
|
