Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: OpenGIS Simple Features Reference Implementation
4 : * Purpose: Implementation of OGC Features and Geometries JSON (JSON-FG)
5 : * Author: Even Rouault <even.rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2023, Even Rouault <even.rouault at spatialys.com>
9 : *
10 : * Permission is hereby granted, free of charge, to any person obtaining a
11 : * copy of this software and associated documentation files (the "Software"),
12 : * to deal in the Software without restriction, including without limitation
13 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 : * and/or sell copies of the Software, and to permit persons to whom the
15 : * Software is furnished to do so, subject to the following conditions:
16 : *
17 : * The above copyright notice and this permission notice shall be included
18 : * in all copies or substantial portions of the Software.
19 : *
20 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 : * DEALINGS IN THE SOFTWARE.
27 : ****************************************************************************/
28 :
29 : #ifndef OGR_JSONFG_H_INCLUDED
30 : #define OGR_JSONFG_H_INCLUDED
31 :
32 : #include "cpl_vsi_virtual.h"
33 :
34 : #include "gdal_priv.h"
35 : #include "ogrsf_frmts.h"
36 : #include "ogrgeojsonutils.h"
37 : #include "ogrgeojsonwriter.h"
38 : #include "ogrjsoncollectionstreamingparser.h"
39 : #include "ogr_mem.h"
40 : #include "directedacyclicgraph.hpp"
41 :
42 : #include <map>
43 : #include <set>
44 : #include <utility>
45 :
46 : /************************************************************************/
47 : /* OGRJSONFGMemLayer */
48 : /************************************************************************/
49 :
50 : /** Layer with all features ingested into memory. */
51 98 : class OGRJSONFGMemLayer final : public OGRMemLayer
52 : {
53 : public:
54 : OGRJSONFGMemLayer(GDALDataset *poDS, const char *pszName,
55 : OGRSpatialReference *poSRS, OGRwkbGeometryType eGType);
56 : ~OGRJSONFGMemLayer();
57 :
58 0 : const char *GetFIDColumn() override
59 : {
60 0 : return osFIDColumn_.c_str();
61 : }
62 :
63 : int TestCapability(const char *pszCap) override;
64 :
65 0 : void SetFIDColumn(const char *pszName)
66 : {
67 0 : osFIDColumn_ = pszName;
68 0 : }
69 :
70 : void AddFeature(std::unique_ptr<OGRFeature> poFeature);
71 :
72 23 : GDALDataset *GetDataset() override
73 : {
74 23 : return m_poDS;
75 : }
76 :
77 : private:
78 : GDALDataset *m_poDS = nullptr;
79 : std::string osFIDColumn_{};
80 : bool bOriginalIdModified_ = false;
81 :
82 : CPL_DISALLOW_COPY_ASSIGN(OGRJSONFGMemLayer)
83 : };
84 :
85 : /************************************************************************/
86 : /* OGRJSONFGStreamedLayer */
87 : /************************************************************************/
88 :
89 : class OGRJSONFGStreamingParser;
90 :
91 : /** Layer with features being acquired progressively through a streaming
92 : parser.
93 :
94 : Only applies for FeatureCollection read through a file
95 : */
96 : class OGRJSONFGStreamedLayer final
97 : : public OGRLayer,
98 : public OGRGetNextFeatureThroughRaw<OGRJSONFGStreamedLayer>
99 : {
100 : public:
101 : OGRJSONFGStreamedLayer(GDALDataset *poDS, const char *pszName,
102 : OGRSpatialReference *poSRS,
103 : OGRwkbGeometryType eGType);
104 : ~OGRJSONFGStreamedLayer();
105 :
106 : // BEGIN specific public API
107 :
108 : //! Set the FID column name
109 0 : void SetFIDColumn(const char *pszName)
110 : {
111 0 : osFIDColumn_ = pszName;
112 0 : }
113 :
114 : //! Set the total feature count
115 92 : void SetFeatureCount(GIntBig nCount)
116 : {
117 92 : nFeatureCount_ = nCount;
118 92 : }
119 :
120 : /** Set the file handle.
121 :
122 : Must be called before GetNextFeature() is called
123 : */
124 : void SetFile(VSIVirtualHandleUniquePtr &&poFile);
125 :
126 : /** Set the streaming parser
127 :
128 : Must be called before GetNextFeature() is called
129 : */
130 : void SetStreamingParser(
131 : std::unique_ptr<OGRJSONFGStreamingParser> &&poStreamingParser);
132 :
133 : // END specific public API
134 :
135 522 : const char *GetFIDColumn() override
136 : {
137 522 : return osFIDColumn_.c_str();
138 : }
139 :
140 2204 : OGRFeatureDefn *GetLayerDefn() override
141 : {
142 2204 : return poFeatureDefn_;
143 : }
144 :
145 : int TestCapability(const char *pszCap) override;
146 :
147 : GIntBig GetFeatureCount(int bForce) override;
148 :
149 : void ResetReading() override;
150 :
151 309 : DEFINE_GET_NEXT_FEATURE_THROUGH_RAW(OGRJSONFGStreamedLayer)
152 :
153 15 : GDALDataset *GetDataset() override
154 : {
155 15 : return m_poDS;
156 : }
157 :
158 : private:
159 : GDALDataset *m_poDS = nullptr;
160 : OGRFeatureDefn *poFeatureDefn_ = nullptr;
161 : std::string osFIDColumn_{};
162 :
163 : /** Total number of features. */
164 : GIntBig nFeatureCount_ = -1;
165 :
166 : VSIVirtualHandleUniquePtr poFile_{};
167 :
168 : std::unique_ptr<OGRJSONFGStreamingParser> poStreamingParser_{};
169 :
170 : /** Whether a warning has been emitted about feature IDs having been
171 : * modified */
172 : bool bOriginalIdModified_ = false;
173 : /** Set of feature IDs read/allocated up to that point */
174 : std::set<GIntBig> oSetUsedFIDs_{};
175 :
176 : /** Ensure the FID of the feature is unique */
177 : OGRFeature *EnsureUniqueFID(OGRFeature *poFeat);
178 :
179 : /** Return next feature (without filter) */
180 : OGRFeature *GetNextRawFeature();
181 :
182 : CPL_DISALLOW_COPY_ASSIGN(OGRJSONFGStreamedLayer)
183 : };
184 :
185 : /************************************************************************/
186 : /* OGRJSONFGWriteLayer */
187 : /************************************************************************/
188 :
189 : class OGRJSONFGDataset;
190 :
191 : class OGRJSONFGWriteLayer final : public OGRLayer
192 : {
193 : public:
194 : OGRJSONFGWriteLayer(
195 : const char *pszName, const OGRSpatialReference *poSRS,
196 : std::unique_ptr<OGRCoordinateTransformation> &&poCTToWGS84,
197 : const std::string &osCoordRefSys, OGRwkbGeometryType eGType,
198 : CSLConstList papszOptions, OGRJSONFGDataset *poDS);
199 : ~OGRJSONFGWriteLayer();
200 :
201 : //
202 : // OGRLayer Interface
203 : //
204 687 : OGRFeatureDefn *GetLayerDefn() override
205 : {
206 687 : return poFeatureDefn_;
207 : }
208 :
209 1 : OGRSpatialReference *GetSpatialRef() override
210 : {
211 1 : return nullptr;
212 : }
213 :
214 16 : void ResetReading() override
215 : {
216 16 : }
217 :
218 16 : OGRFeature *GetNextFeature() override
219 : {
220 16 : return nullptr;
221 : }
222 :
223 : OGRErr ICreateFeature(OGRFeature *poFeature) override;
224 : OGRErr CreateField(const OGRFieldDefn *poField, int bApproxOK) override;
225 : int TestCapability(const char *pszCap) override;
226 :
227 : OGRErr SyncToDisk() override;
228 :
229 : GDALDataset *GetDataset() override;
230 :
231 : private:
232 : OGRJSONFGDataset *poDS_{};
233 : OGRFeatureDefn *poFeatureDefn_ = nullptr;
234 : std::unique_ptr<OGRCoordinateTransformation> poCTToWGS84_;
235 : bool bIsWGS84CRS_ = false;
236 : bool m_bMustSwapForPlace = false;
237 : int nOutCounter_ = 0;
238 : std::string osCoordRefSys_{};
239 :
240 : OGRGeoJSONWriteOptions oWriteOptions_{};
241 : OGRGeoJSONWriteOptions oWriteOptionsPlace_{};
242 : bool bWriteFallbackGeometry_ = true;
243 :
244 : CPL_DISALLOW_COPY_ASSIGN(OGRJSONFGWriteLayer)
245 : };
246 :
247 : /************************************************************************/
248 : /* OGRJSONFGDataset */
249 : /************************************************************************/
250 :
251 : class OGRJSONFGReader;
252 :
253 : class OGRJSONFGDataset final : public GDALDataset
254 : {
255 : public:
256 211 : OGRJSONFGDataset() = default;
257 : ~OGRJSONFGDataset();
258 :
259 : bool Open(GDALOpenInfo *poOpenInfo, GeoJSONSourceType nSrcType);
260 : bool Create(const char *pszName, CSLConstList papszOptions);
261 :
262 43 : int GetLayerCount() override
263 : {
264 43 : return static_cast<int>(apoLayers_.size());
265 : }
266 :
267 : OGRLayer *GetLayer(int i) override;
268 :
269 : //! Return the output file handle. Used by OGRJSONFGWriteLayer
270 230 : VSILFILE *GetOutputFile() const
271 : {
272 230 : return fpOut_;
273 : }
274 :
275 : /** Return whether there is a single output layer.
276 : * Used by OGRJSONFGWriteLayer
277 : */
278 240 : bool IsSingleOutputLayer() const
279 : {
280 240 : return bSingleOutputLayer_;
281 : }
282 :
283 : //! Return whether the output file is seekable
284 2 : bool GetFpOutputIsSeekable() const
285 : {
286 2 : return bFpOutputIsSeekable_;
287 : }
288 :
289 : void BeforeCreateFeature();
290 :
291 : OGRLayer *ICreateLayer(const char *pszName,
292 : const OGRGeomFieldDefn *poGeomFieldDefn,
293 : CSLConstList papszOptions) override;
294 :
295 : int TestCapability(const char *pszCap) override;
296 :
297 : OGRErr SyncToDiskInternal();
298 :
299 : protected:
300 : friend class OGRJSONFGReader;
301 : OGRJSONFGMemLayer *AddLayer(std::unique_ptr<OGRJSONFGMemLayer> &&poLayer);
302 : OGRJSONFGStreamedLayer *
303 : AddLayer(std::unique_ptr<OGRJSONFGStreamedLayer> &&poLayer);
304 :
305 : private:
306 : char *pszGeoData_ = nullptr;
307 : size_t nGeoDataLen_ = 0;
308 : std::vector<std::unique_ptr<OGRLayer>> apoLayers_{};
309 : std::unique_ptr<OGRJSONFGReader> poReader_{};
310 :
311 : // Write side
312 : VSILFILE *fpOut_ = nullptr;
313 : bool bSingleOutputLayer_ = false;
314 : bool bHasEmittedFeatures_ = false;
315 : bool bFpOutputIsSeekable_ = false;
316 :
317 : /** Offset at which the '] }' terminating sequence has already been
318 : * written by SyncToDisk(). 0 if it has not been written.
319 : */
320 : vsi_l_offset m_nPositionBeforeFCClosed = 0;
321 :
322 : bool ReadFromFile(GDALOpenInfo *poOpenInfo, const char *pszUnprefixed);
323 : bool ReadFromService(GDALOpenInfo *poOpenInfo, const char *pszSource);
324 :
325 : void FinishWriting();
326 :
327 : bool EmitStartFeaturesIfNeededAndReturnIfFirstFeature();
328 :
329 : CPL_DISALLOW_COPY_ASSIGN(OGRJSONFGDataset)
330 : };
331 :
332 : /************************************************************************/
333 : /* OGRJSONFGReader */
334 : /************************************************************************/
335 :
336 : class OGRJSONFGReader
337 : {
338 : public:
339 141 : OGRJSONFGReader() = default;
340 : ~OGRJSONFGReader();
341 :
342 : /** Load all features from the passed in JSON text in OGRJSONFGMemLayer(s)
343 : *
344 : * This method should only be called once, and is exclusive with
345 : * AnalyzeWithStreamingParser()
346 : */
347 : bool Load(OGRJSONFGDataset *poDS, const char *pszText,
348 : const std::string &osDefaultLayerName);
349 :
350 : /** Do a first pass analysis of the content of the passed file to create
351 : * OGRJSONFGStreamedLayer's
352 : *
353 : * It is the responsibility of the caller to call
354 : * SetFile() and SetStreamingParser() on the created layers afterwards
355 : *
356 : * This method should only be called once, and is exclusive with
357 : * Load()
358 : */
359 : bool AnalyzeWithStreamingParser(OGRJSONFGDataset *poDS, VSILFILE *fp,
360 : const std::string &osDefaultLayerName,
361 : bool &bCanTryWithNonStreamingParserOut);
362 :
363 : /** Geometry element we are interested in. */
364 : enum class GeometryElement
365 : {
366 : /** Use "place" when possible, fallback to "geometry" otherwise. */
367 : AUTO,
368 : /** Only use "place" */
369 : PLACE,
370 : /** Only use "geometry" */
371 : GEOMETRY,
372 : };
373 :
374 : /** Sets the geometry element we are interested in. */
375 6 : void SetGeometryElement(GeometryElement elt)
376 : {
377 6 : eGeometryElement_ = elt;
378 6 : }
379 :
380 : /** Returns a OGRFeature built from the passed in JSON object.
381 : *
382 : * @param poObj JSON feature
383 : * @param pszRequestedLayer name of the layer of interest, or nullptr if
384 : * no filtering needed on the layer name. If the feature does not belong
385 : * to the requested layer, nullptr is returned.
386 : * @param pOutMemLayer Pointer to the OGRJSONFGMemLayer* layer to which
387 : * the returned feature belongs to. May be nullptr. Only applies when
388 : * the Load() method has been used.
389 : * @param pOutStreamedLayer Pointer to the OGRJSONFGStreamedLayer* layer to
390 : * which the returned feature belongs to. May be nullptr. Only applies when
391 : * the AnalyzeWithStreamingParser() method has been used.
392 : */
393 : std::unique_ptr<OGRFeature>
394 : ReadFeature(json_object *poObj, const char *pszRequestedLayer,
395 : OGRJSONFGMemLayer **pOutMemLayer,
396 : OGRJSONFGStreamedLayer **pOutStreamedLayer);
397 :
398 : protected:
399 : friend class OGRJSONFGStreamingParser;
400 :
401 : bool GenerateLayerDefnFromFeature(json_object *poObj);
402 :
403 : private:
404 : GeometryElement eGeometryElement_ = GeometryElement::AUTO;
405 :
406 : OGRJSONFGDataset *poDS_ = nullptr;
407 : std::string osDefaultLayerName_{};
408 : json_object *poObject_ = nullptr;
409 :
410 : bool bFlattenNestedAttributes_ = false;
411 : char chNestedAttributeSeparator_ = 0;
412 : bool bArrayAsString_ = false;
413 : bool bDateAsString_ = false;
414 :
415 : /** Layer building context, specific to one layer. */
416 153 : struct LayerDefnBuildContext
417 : {
418 : //! Maps a field name to its index in apoFieldDefn[]
419 : std::map<std::string, int> oMapFieldNameToIdx{};
420 :
421 : //! Vector of OGRFieldDefn
422 : std::vector<std::unique_ptr<OGRFieldDefn>> apoFieldDefn{};
423 :
424 : //! Directed acyclic graph used to build the order of fields.
425 : gdal::DirectedAcyclicGraph<int, std::string> dag{};
426 :
427 : /** Set of indices of apoFieldDefn[] for which no type information is
428 : * known yet. */
429 : std::set<int> aoSetUndeterminedTypeFields{};
430 :
431 : //! Whether at least one feature has a "coordRefSys" member.
432 : bool bHasCoordRefSysAtFeatureLevel = false;
433 :
434 : /** CRS object corresponding to "coordRefsys" member at feature level.
435 : * Only set if homogeneous among features.
436 : */
437 : std::unique_ptr<OGRSpatialReference> poCRSAtFeatureLevel{};
438 :
439 : /** Serialized JSON value of "coordRefsys" member at feature level.
440 : * Only set if homogeneous among features.
441 : */
442 : std::string osCoordRefSysAtFeatureLevel{};
443 :
444 : /** Whether to switch X/Y ordinates in geometries appearing in "place"
445 : * element. Only applies to CRS at layer level.
446 : */
447 : bool bSwapPlacesXY = false;
448 :
449 : //! Whether the layer CRS is WGS 84.
450 : bool bLayerCRSIsWGS84 = false;
451 :
452 : //! Coordinate transformation from WGS 84 to layer CRS (might be null)
453 : std::unique_ptr<OGRCoordinateTransformation> poCTWGS84ToLayerCRS{};
454 :
455 : /** Feature count */
456 : GIntBig nFeatureCount = 0;
457 :
458 : //! Whether the Feature.id should be mapped to a OGR field.
459 : bool bFeatureLevelIdAsAttribute = false;
460 :
461 : //! Whether the Feature.id should be mapped to a OGR FID.
462 : bool bFeatureLevelIdAsFID = false;
463 :
464 : //! Whether 64-bit integers are needed for OGR FID.
465 : bool bNeedFID64 = false;
466 :
467 : //! Whether detection of layer geometry type is still needed.
468 : bool bDetectLayerGeomType = true;
469 :
470 : //! Whether no geometry has been analyzed yet.
471 : bool bFirstGeometry = true;
472 :
473 : //! Layer geometry type.
474 : OGRwkbGeometryType eLayerGeomType = wkbUnknown;
475 :
476 : //! Whether a Feature.time.date element has been found.
477 : bool bHasTimeDate = false;
478 :
479 : //! Whether a Feature.time.timestamp element has been found.
480 : bool bHasTimeTimestamp = false;
481 :
482 : /** Whether a Feature.time.interval[0] element of type timestamp has
483 : * been found */
484 : bool bHasTimeIntervalStartTimestamp = false;
485 :
486 : /** Whether a Feature.time.interval[0] element of type date has
487 : * been found */
488 : bool bHasTimeIntervalStartDate = false;
489 :
490 : /** Whether a Feature.time.interval[1] element of type timestamp has
491 : * been found */
492 : bool bHasTimeIntervalEndTimestamp = false;
493 :
494 : /** Whether a Feature.time.interval[1] element of type date has
495 : * been found */
496 : bool bHasTimeIntervalEndDate = false;
497 :
498 : //! Index of OGR field "time" / "jsonfg_time"
499 : int nIdxFieldTime = -1;
500 :
501 : //! Index of OGR field "time_start" / "jsonfg_time_start"
502 : int nIdxFieldTimeStart = -1;
503 :
504 : //! Index of OGR field "time_end" / "jsonfg_time_end"
505 : int nIdxFieldTimeEnd = -1;
506 :
507 : //! Corresponding OGRJSONFGMemLayer (only for Load() ingestion mode)
508 : OGRJSONFGMemLayer *poMemLayer = nullptr;
509 :
510 : /** Corresponding OGRJSONFGStreamedLayer(only for
511 : * AnalyzeWithStreamingParser() mode) */
512 : OGRJSONFGStreamedLayer *poStreamedLayer = nullptr;
513 :
514 294 : LayerDefnBuildContext() = default;
515 12 : LayerDefnBuildContext(LayerDefnBuildContext &&) = default;
516 : LayerDefnBuildContext &operator=(LayerDefnBuildContext &&) = default;
517 :
518 : private:
519 : CPL_DISALLOW_COPY_ASSIGN(LayerDefnBuildContext)
520 : };
521 :
522 : //! Maps a layer name to its build context
523 : std::map<std::string, LayerDefnBuildContext> oMapBuildContext_{};
524 :
525 : //
526 : // Copy operations not supported.
527 : //
528 : CPL_DISALLOW_COPY_ASSIGN(OGRJSONFGReader)
529 :
530 : const char *GetLayerNameForFeature(json_object *poObj) const;
531 : bool GenerateLayerDefns();
532 : bool FinalizeGenerateLayerDefns(bool bStreamedLayer);
533 : void FinalizeBuildContext(LayerDefnBuildContext &oBuildContext,
534 : const char *pszLayerName, bool bStreamedLayer,
535 : bool bInvalidCRS, bool bSwapPlacesXYTopLevel,
536 : OGRSpatialReference *poSRSTopLevel);
537 : };
538 :
539 : /************************************************************************/
540 : /* OGRJSONFGStreamingParser */
541 : /************************************************************************/
542 :
543 : /** FeatureCollection streaming parser. */
544 478 : class OGRJSONFGStreamingParser final : public OGRJSONCollectionStreamingParser
545 : {
546 : OGRJSONFGReader &m_oReader;
547 : std::string m_osRequestedLayer{};
548 :
549 : std::vector<std::pair<std::unique_ptr<OGRFeature>, OGRLayer *>>
550 : m_apoFeatures{};
551 : size_t m_nCurFeatureIdx = 0;
552 :
553 : CPL_DISALLOW_COPY_ASSIGN(OGRJSONFGStreamingParser)
554 :
555 : protected:
556 : void GotFeature(json_object *poObj, bool bFirstPass,
557 : const std::string &osJson) override;
558 : void TooComplex() override;
559 :
560 : public:
561 : OGRJSONFGStreamingParser(OGRJSONFGReader &oReader, bool bFirstPass);
562 : ~OGRJSONFGStreamingParser();
563 :
564 92 : void SetRequestedLayer(const char *pszRequestedLayer)
565 : {
566 92 : m_osRequestedLayer = pszRequestedLayer;
567 92 : }
568 :
569 : std::unique_ptr<OGRJSONFGStreamingParser> Clone();
570 :
571 : std::pair<std::unique_ptr<OGRFeature>, OGRLayer *> GetNextFeature();
572 : };
573 :
574 : bool OGRJSONFGMustSwapXY(const OGRSpatialReference *poSRS);
575 :
576 : #endif // OGR_JSONFG_H_INCLUDED
|
