Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: GDAL
4 : * Purpose: gdal "vector pipeline" subcommand
5 : * Author: Even Rouault <even dot rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2024, Even Rouault <even dot rouault at spatialys.com>
9 : *
10 : * SPDX-License-Identifier: MIT
11 : ****************************************************************************/
12 :
13 : #ifndef GDALALG_VECTOR_PIPELINE_INCLUDED
14 : #define GDALALG_VECTOR_PIPELINE_INCLUDED
15 :
16 : #include "gdalalgorithm.h"
17 : #include "gdalalg_abstract_pipeline.h"
18 :
19 : #include "ogrsf_frmts.h"
20 : #include "ogrlayerwithtranslatefeature.h"
21 :
22 : #include <map>
23 : #include <tuple>
24 : #include <vector>
25 :
26 : //! @cond Doxygen_Suppress
27 :
28 : /************************************************************************/
29 : /* GDALVectorPipelineStepAlgorithm */
30 : /************************************************************************/
31 :
32 : class GDALRasterAlgorithmStepRegistry;
33 :
34 4630 : class GDALVectorPipelineStepAlgorithm /* non final */
35 : : public GDALPipelineStepAlgorithm
36 : {
37 : public:
38 : ~GDALVectorPipelineStepAlgorithm() override;
39 :
40 : protected:
41 : GDALVectorPipelineStepAlgorithm(const std::string &name,
42 : const std::string &description,
43 : const std::string &helpURL,
44 : bool standaloneStep);
45 :
46 : GDALVectorPipelineStepAlgorithm(const std::string &name,
47 : const std::string &description,
48 : const std::string &helpURL,
49 : const ConstructorOptions &options);
50 :
51 : friend class GDALVectorPipelineAlgorithm;
52 : friend class GDALVectorConcatAlgorithm;
53 :
54 523 : int GetInputType() const override
55 : {
56 523 : return GDAL_OF_VECTOR;
57 : }
58 :
59 479 : int GetOutputType() const override
60 : {
61 479 : return GDAL_OF_VECTOR;
62 : }
63 : };
64 :
65 : /************************************************************************/
66 : /* GDALVectorAlgorithmStepRegistry */
67 : /************************************************************************/
68 :
69 410 : class GDALVectorAlgorithmStepRegistry : public virtual GDALAlgorithmRegistry
70 : {
71 : public:
72 410 : GDALVectorAlgorithmStepRegistry() = default;
73 : ~GDALVectorAlgorithmStepRegistry() override;
74 :
75 : /** Register the algorithm of type MyAlgorithm.
76 : */
77 : template <class MyAlgorithm>
78 14905 : bool Register(const std::string &name = std::string())
79 : {
80 : static_assert(
81 : std::is_base_of_v<GDALVectorPipelineStepAlgorithm, MyAlgorithm>,
82 : "Algorithm is not a GDALVectorPipelineStepAlgorithm");
83 :
84 29810 : AlgInfo info;
85 14905 : info.m_name = name.empty() ? MyAlgorithm::NAME : name;
86 14905 : info.m_aliases = MyAlgorithm::GetAliasesStatic();
87 16490 : info.m_creationFunc = []() -> std::unique_ptr<GDALAlgorithm>
88 1585 : { return std::make_unique<MyAlgorithm>(); };
89 29810 : return GDALAlgorithmRegistry::Register(info);
90 : }
91 : };
92 :
93 : /************************************************************************/
94 : /* GDALVectorPipelineAlgorithm */
95 : /************************************************************************/
96 :
97 : class GDALVectorPipelineAlgorithm final : public GDALAbstractPipelineAlgorithm
98 : {
99 : public:
100 : static constexpr const char *NAME = "pipeline";
101 : static constexpr const char *DESCRIPTION =
102 : "Process a vector dataset applying several steps.";
103 : static constexpr const char *HELP_URL =
104 : "/programs/gdal_vector_pipeline.html";
105 :
106 968 : static std::vector<std::string> GetAliasesStatic()
107 : {
108 : return {
109 : #ifdef GDAL_PIPELINE_PROJ_NOSTALGIA
110 : GDALAlgorithmRegistry::HIDDEN_ALIAS_SEPARATOR,
111 : "+pipeline",
112 : "+gdal=pipeline",
113 : #endif
114 3872 : };
115 : }
116 :
117 : GDALVectorPipelineAlgorithm();
118 :
119 : std::string GetUsageForCLI(bool shortUsage,
120 : const UsageOptions &usageOptions) const override;
121 :
122 : static void RegisterAlgorithms(GDALVectorAlgorithmStepRegistry ®istry,
123 : bool forMixedPipeline);
124 :
125 200 : int GetInputType() const override
126 : {
127 200 : return GDAL_OF_VECTOR;
128 : }
129 :
130 0 : int GetOutputType() const override
131 : {
132 0 : return GDAL_OF_VECTOR;
133 : }
134 :
135 : protected:
136 : GDALVectorAlgorithmStepRegistry m_stepRegistry{};
137 :
138 214 : GDALAlgorithmRegistry &GetStepRegistry() override
139 : {
140 214 : return m_stepRegistry;
141 : }
142 :
143 820 : const GDALAlgorithmRegistry &GetStepRegistry() const override
144 : {
145 820 : return m_stepRegistry;
146 : }
147 :
148 : private:
149 : std::unique_ptr<GDALAbstractPipelineAlgorithm>
150 0 : CreateNestedPipeline() const override
151 : {
152 0 : auto pipeline = std::make_unique<GDALVectorPipelineAlgorithm>();
153 0 : pipeline->m_bInnerPipeline = true;
154 0 : return pipeline;
155 : }
156 : };
157 :
158 : /************************************************************************/
159 : /* GDALVectorDecoratedDataset */
160 : /************************************************************************/
161 :
162 : /** Base class for GDALVectorOutputDataset, GDALVectorNonStreamingAlgorithmDataset
163 : * and GDALVectorPipelineOutputDataset
164 : */
165 317 : class GDALVectorDecoratedDataset /* non final */
166 : : public GDALDataset
167 : {
168 : public:
169 : ~GDALVectorDecoratedDataset() override;
170 :
171 9 : CSLConstList GetMetadata(const char *pszDomain) override
172 : {
173 9 : return m_srcDS.GetMetadata(pszDomain);
174 : }
175 :
176 9 : const char *GetMetadataItem(const char *pszName,
177 : const char *pszDomain) override
178 : {
179 9 : return m_srcDS.GetMetadataItem(pszName, pszDomain);
180 : }
181 :
182 : std::vector<std::string>
183 2 : GetFieldDomainNames(CSLConstList papszOptions) const override
184 : {
185 2 : return m_srcDS.GetFieldDomainNames(papszOptions);
186 : }
187 :
188 1 : const OGRFieldDomain *GetFieldDomain(const std::string &name) const override
189 : {
190 1 : return m_srcDS.GetFieldDomain(name);
191 : }
192 :
193 : std::vector<std::string>
194 104 : GetRelationshipNames(CSLConstList papszOptions) const override
195 : {
196 104 : return m_srcDS.GetRelationshipNames(papszOptions);
197 : }
198 :
199 : const GDALRelationship *
200 1 : GetRelationship(const std::string &name) const override
201 : {
202 1 : return m_srcDS.GetRelationship(name);
203 : }
204 :
205 : private:
206 : std::unique_ptr<GDALDataset> m_dummySrcDS{};
207 :
208 : protected:
209 : explicit GDALVectorDecoratedDataset(GDALDataset *poSrcDS);
210 :
211 : GDALDataset &m_srcDS;
212 : };
213 :
214 : /************************************************************************/
215 : /* GDALVectorOutputDataset */
216 : /************************************************************************/
217 :
218 : class GDALVectorOutputDataset final : public GDALVectorDecoratedDataset
219 : {
220 :
221 : public:
222 : explicit GDALVectorOutputDataset(GDALDataset *poSrcDS);
223 :
224 32 : int GetLayerCount() const override
225 : {
226 32 : return static_cast<int>(m_layers.size());
227 : }
228 :
229 17 : const OGRLayer *GetLayer(int idx) const override
230 : {
231 17 : return m_layers[idx].get();
232 : }
233 :
234 : int TestCapability(const char *) const override;
235 :
236 17 : void AddLayer(std::unique_ptr<OGRLayer> layer)
237 : {
238 17 : m_layers.emplace_back(std::move(layer));
239 17 : }
240 :
241 : private:
242 : std::vector<std::unique_ptr<OGRLayer>> m_layers{};
243 : };
244 :
245 : /************************************************************************/
246 : /* GDALVectorAlgorithmLayerProgressHelper */
247 : /************************************************************************/
248 :
249 : /**
250 : * This class helps doing progress report for algorithm iterating over layers
251 : * of the source dataset.
252 : */
253 : class GDALVectorAlgorithmLayerProgressHelper
254 : {
255 : public:
256 : /** Constructor */
257 : GDALVectorAlgorithmLayerProgressHelper(GDALProgressFunc pfnProgress,
258 : void *pProgressData);
259 : /** Constructor */
260 : explicit GDALVectorAlgorithmLayerProgressHelper(
261 : const GDALPipelineStepRunContext &ctxt);
262 :
263 : /** Register the passed layer as a layer that will be processed. */
264 : void AddProcessedLayer(OGRLayer &srcLayer);
265 :
266 : /** Register the passed layer as a layer that will be forwarded without
267 : * processing. */
268 : void AddPassThroughLayer(OGRLayer &srcLayer);
269 :
270 : //! @cond Doxygen_Suppress
271 : class iterator
272 : {
273 : public:
274 140 : explicit iterator(const GDALVectorAlgorithmLayerProgressHelper &helper,
275 : bool start)
276 140 : : m_helper(helper),
277 140 : m_nLayerIdx(start ? 0 : m_helper.m_apoSrcLayers.size())
278 : {
279 140 : }
280 :
281 : inline bool operator==(const iterator &other) const
282 : {
283 : return m_nLayerIdx == other.m_nLayerIdx;
284 : }
285 :
286 137 : inline bool operator!=(const iterator &other) const
287 : {
288 137 : return m_nLayerIdx != other.m_nLayerIdx;
289 : }
290 :
291 67 : inline iterator &operator++()
292 : {
293 67 : if (!m_helper.m_anFeatures.empty())
294 63 : m_nFeatureIdx += m_helper.m_anFeatures[m_nProcessedLayerIdx];
295 67 : if (m_helper.m_apoSrcLayers[m_nLayerIdx].second)
296 61 : ++m_nProcessedLayerIdx;
297 67 : ++m_nLayerIdx;
298 67 : return *this;
299 : }
300 :
301 : using progress_data_unique_ptr =
302 : std::unique_ptr<void, decltype(&GDALDestroyScaledProgress)>;
303 : using value_type = std::tuple<OGRLayer *, bool, GDALProgressFunc,
304 : progress_data_unique_ptr>;
305 :
306 : value_type operator*() const;
307 :
308 : private:
309 : const GDALVectorAlgorithmLayerProgressHelper &m_helper;
310 : size_t m_nLayerIdx = 0;
311 : size_t m_nProcessedLayerIdx = 0;
312 : GIntBig m_nFeatureIdx = 0;
313 : };
314 :
315 : //! @endcond
316 :
317 : /** Start of an iterator over layers registered with AddProcessedLayer()
318 : * and AddUnprocessedLayer() */
319 70 : iterator begin() const
320 : {
321 70 : return iterator(*this, true);
322 : }
323 :
324 : /** End of an iterator over layers registered with AddProcessedLayer()
325 : * and AddUnprocessedLayer() */
326 70 : iterator end() const
327 : {
328 70 : return iterator(*this, false);
329 : }
330 :
331 : /** Return if AddProcessedLayer() has been called at least once. */
332 22 : bool HasProcessedLayers() const
333 : {
334 22 : return !m_anFeatures.empty();
335 : }
336 :
337 : private:
338 : GDALProgressFunc m_pfnProgress = nullptr;
339 : void *m_pProgressData = nullptr;
340 : int64_t m_nTotalFeatures = 0;
341 : std::vector<std::pair<OGRLayer *, bool>> m_apoSrcLayers{};
342 : std::vector<int64_t> m_anFeatures{};
343 :
344 : CPL_DISALLOW_COPY_ASSIGN(GDALVectorAlgorithmLayerProgressHelper)
345 : };
346 :
347 : /************************************************************************/
348 : /* GDALVectorPipelineOutputLayer */
349 : /************************************************************************/
350 :
351 : /** Class that implements GetNextFeature() by forwarding to
352 : * OGRLayerWithTranslateFeature::TranslateFeature() implementation, which
353 : * might return several features.
354 : */
355 255 : class GDALVectorPipelineOutputLayer /* non final */
356 : : public OGRLayerWithTranslateFeature,
357 : public OGRGetNextFeatureThroughRaw<GDALVectorPipelineOutputLayer>
358 : {
359 : protected:
360 : explicit GDALVectorPipelineOutputLayer(OGRLayer &oSrcLayer);
361 : ~GDALVectorPipelineOutputLayer() override;
362 :
363 2620 : DEFINE_GET_NEXT_FEATURE_THROUGH_RAW(GDALVectorPipelineOutputLayer)
364 :
365 : OGRLayer &m_srcLayer;
366 :
367 7 : void FailTranslation()
368 : {
369 7 : m_translateError = true;
370 7 : }
371 :
372 : public:
373 : void ResetReading() override;
374 : OGRFeature *GetNextRawFeature();
375 :
376 : private:
377 : std::vector<std::unique_ptr<OGRFeature>> m_pendingFeatures{};
378 : size_t m_idxInPendingFeatures = 0;
379 : bool m_translateError = false;
380 : };
381 :
382 : /************************************************************************/
383 : /* GDALVectorPipelinePassthroughLayer */
384 : /************************************************************************/
385 :
386 : /** Class that forwards GetNextFeature() calls to the source layer and
387 : * can be added to GDALVectorPipelineOutputDataset::AddLayer()
388 : */
389 : class GDALVectorPipelinePassthroughLayer /* non final */
390 : : public GDALVectorPipelineOutputLayer
391 : {
392 : public:
393 29 : explicit GDALVectorPipelinePassthroughLayer(OGRLayer &oSrcLayer)
394 29 : : GDALVectorPipelineOutputLayer(oSrcLayer)
395 : {
396 29 : }
397 :
398 : const OGRFeatureDefn *GetLayerDefn() const override;
399 :
400 4 : int TestCapability(const char *pszCap) const override
401 : {
402 4 : return m_srcLayer.TestCapability(pszCap);
403 : }
404 :
405 1 : OGRErr IGetExtent(int iGeomField, OGREnvelope *psExtent,
406 : bool bForce) override
407 : {
408 1 : return m_srcLayer.GetExtent(iGeomField, psExtent, bForce);
409 : }
410 :
411 1 : OGRErr IGetExtent3D(int iGeomField, OGREnvelope3D *psExtent,
412 : bool bForce) override
413 : {
414 1 : return m_srcLayer.GetExtent3D(iGeomField, psExtent, bForce);
415 : }
416 :
417 190 : void TranslateFeature(
418 : std::unique_ptr<OGRFeature> poSrcFeature,
419 : std::vector<std::unique_ptr<OGRFeature>> &apoOutFeatures) override
420 : {
421 190 : apoOutFeatures.push_back(std::move(poSrcFeature));
422 190 : }
423 : };
424 :
425 : /************************************************************************/
426 : /* GDALVectorNonStreamingAlgorithmLayer */
427 : /************************************************************************/
428 :
429 : /**
430 : * This class represents a layer for algorithms that process vector data
431 : * in a non-streaming manner.
432 : *
433 : * Implementations must override the following methods:
434 : * - Process(), which is called when the first feature is read
435 : * - GetNextProcessedFeature() method which provides features in sequential order
436 : */
437 67 : class GDALVectorNonStreamingAlgorithmLayer
438 : : public OGRLayer,
439 : public OGRGetNextFeatureThroughRaw<GDALVectorNonStreamingAlgorithmLayer>
440 : {
441 : public:
442 : GDALVectorNonStreamingAlgorithmLayer(OGRLayer &srcLayer,
443 : int geomFieldIndex);
444 :
445 : ~GDALVectorNonStreamingAlgorithmLayer() override;
446 :
447 4 : const char *GetDescription() const override
448 : {
449 4 : return GetName();
450 : }
451 :
452 : virtual bool Process(GDALProgressFunc pfnProgress, void *pProgressData) = 0;
453 :
454 : virtual std::unique_ptr<OGRFeature> GetNextProcessedFeature() = 0;
455 :
456 : OGRFeature *GetNextRawFeature();
457 :
458 1876 : DEFINE_GET_NEXT_FEATURE_THROUGH_RAW(GDALVectorNonStreamingAlgorithmLayer)
459 :
460 : protected:
461 : OGRLayer &m_srcLayer;
462 : int m_geomFieldIndex{0};
463 :
464 : private:
465 : CPL_DISALLOW_COPY_ASSIGN(GDALVectorNonStreamingAlgorithmLayer)
466 : };
467 :
468 : /************************************************************************/
469 : /* GDALVectorNonStreamingAlgorithmDataset */
470 : /************************************************************************/
471 :
472 : /**
473 : * Dataset used to read all input features into memory and perform some
474 : * processing.
475 : */
476 148 : class GDALVectorNonStreamingAlgorithmDataset /* non final */
477 : : public GDALVectorDecoratedDataset
478 : {
479 : public:
480 : explicit GDALVectorNonStreamingAlgorithmDataset(GDALDataset &oSrcDS);
481 : ~GDALVectorNonStreamingAlgorithmDataset() override;
482 :
483 : /** Add a layer to the dataset and perform the associated processing. */
484 : bool AddProcessedLayer(
485 : std::unique_ptr<GDALVectorNonStreamingAlgorithmLayer> srcLayer,
486 : GDALProgressFunc progressFunc, void *progressData);
487 :
488 : void AddPassThroughLayer(OGRLayer &oLayer);
489 :
490 : int GetLayerCount() const final override;
491 : OGRLayer *GetLayer(int idx) const final override;
492 : int TestCapability(const char *pszCap) const override;
493 :
494 : private:
495 : std::vector<std::unique_ptr<OGRLayer>> m_owned_layers{};
496 : std::vector<OGRLayer *> m_layers{};
497 : };
498 :
499 : /************************************************************************/
500 : /* GDALVectorPipelineOutputDataset */
501 : /************************************************************************/
502 :
503 : /** Class used by vector pipeline steps to create an output on-the-fly
504 : * dataset where they can store on-the-fly layers.
505 : */
506 : class GDALVectorPipelineOutputDataset /* non final */
507 : : public GDALVectorDecoratedDataset
508 : {
509 : std::map<OGRLayer *, OGRLayerWithTranslateFeature *>
510 : m_mapSrcLayerToNewLayer{};
511 : std::vector<std::unique_ptr<OGRLayerWithTranslateFeature>>
512 : m_layersToDestroy{};
513 : std::vector<OGRLayerWithTranslateFeature *> m_layers{};
514 :
515 : OGRLayerWithTranslateFeature *m_belongingLayer = nullptr;
516 : std::vector<std::unique_ptr<OGRFeature>> m_pendingFeatures{};
517 : size_t m_idxInPendingFeatures = 0;
518 :
519 : CPL_DISALLOW_COPY_ASSIGN(GDALVectorPipelineOutputDataset)
520 :
521 : public:
522 : explicit GDALVectorPipelineOutputDataset(GDALDataset &oSrcDS);
523 :
524 : void AddLayer(OGRLayer &oSrcLayer,
525 : std::unique_ptr<OGRLayerWithTranslateFeature> poNewLayer);
526 :
527 : int GetLayerCount() const override;
528 :
529 : OGRLayer *GetLayer(int idx) const override;
530 :
531 : int TestCapability(const char *pszCap) const override;
532 :
533 : void ResetReading() override;
534 :
535 : OGRFeature *GetNextFeature(OGRLayer **ppoBelongingLayer,
536 : double *pdfProgressPct,
537 : GDALProgressFunc pfnProgress,
538 : void *pProgressData) override;
539 : };
540 :
541 : //! @endcond
542 :
543 : #endif
|
