Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: GDAL
4 : * Purpose: gdal "vector pipeline" subcommand
5 : * Author: Even Rouault <even dot rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2024, Even Rouault <even dot rouault at spatialys.com>
9 : *
10 : * SPDX-License-Identifier: MIT
11 : ****************************************************************************/
12 :
13 : #ifndef GDALALG_VECTOR_PIPELINE_INCLUDED
14 : #define GDALALG_VECTOR_PIPELINE_INCLUDED
15 :
16 : #include "gdalalgorithm.h"
17 : #include "gdalalg_abstract_pipeline.h"
18 :
19 : #include "ogrsf_frmts.h"
20 : #include "ogrlayerwithtranslatefeature.h"
21 :
22 : #include <map>
23 : #include <tuple>
24 : #include <vector>
25 :
26 : //! @cond Doxygen_Suppress
27 :
28 : /************************************************************************/
29 : /* GDALVectorPipelineStepAlgorithm */
30 : /************************************************************************/
31 :
32 : class GDALRasterAlgorithmStepRegistry;
33 :
34 2590 : class GDALVectorPipelineStepAlgorithm /* non final */
35 : : public GDALPipelineStepAlgorithm
36 : {
37 : public:
38 : ~GDALVectorPipelineStepAlgorithm() override;
39 :
40 : protected:
41 : GDALVectorPipelineStepAlgorithm(const std::string &name,
42 : const std::string &description,
43 : const std::string &helpURL,
44 : bool standaloneStep);
45 :
46 : GDALVectorPipelineStepAlgorithm(const std::string &name,
47 : const std::string &description,
48 : const std::string &helpURL,
49 : const ConstructorOptions &options);
50 :
51 : friend class GDALVectorPipelineAlgorithm;
52 : friend class GDALVectorConcatAlgorithm;
53 :
54 372 : int GetInputType() const override
55 : {
56 372 : return GDAL_OF_VECTOR;
57 : }
58 :
59 356 : int GetOutputType() const override
60 : {
61 356 : return GDAL_OF_VECTOR;
62 : }
63 : };
64 :
65 : /************************************************************************/
66 : /* GDALVectorAlgorithmStepRegistry */
67 : /************************************************************************/
68 :
69 312 : class GDALVectorAlgorithmStepRegistry : public virtual GDALAlgorithmRegistry
70 : {
71 : public:
72 312 : GDALVectorAlgorithmStepRegistry() = default;
73 : ~GDALVectorAlgorithmStepRegistry() override;
74 :
75 : /** Register the algorithm of type MyAlgorithm.
76 : */
77 : template <class MyAlgorithm>
78 9048 : bool Register(const std::string &name = std::string())
79 : {
80 : static_assert(
81 : std::is_base_of_v<GDALVectorPipelineStepAlgorithm, MyAlgorithm>,
82 : "Algorithm is not a GDALVectorPipelineStepAlgorithm");
83 :
84 18096 : AlgInfo info;
85 9048 : info.m_name = name.empty() ? MyAlgorithm::NAME : name;
86 9048 : info.m_aliases = MyAlgorithm::GetAliasesStatic();
87 10331 : info.m_creationFunc = []() -> std::unique_ptr<GDALAlgorithm>
88 1283 : { return std::make_unique<MyAlgorithm>(); };
89 18096 : return GDALAlgorithmRegistry::Register(info);
90 : }
91 : };
92 :
93 : /************************************************************************/
94 : /* GDALVectorPipelineAlgorithm */
95 : /************************************************************************/
96 :
97 : class GDALVectorPipelineAlgorithm final : public GDALAbstractPipelineAlgorithm
98 : {
99 : public:
100 : static constexpr const char *NAME = "pipeline";
101 : static constexpr const char *DESCRIPTION =
102 : "Process a vector dataset applying several steps.";
103 : static constexpr const char *HELP_URL =
104 : "/programs/gdal_vector_pipeline.html";
105 :
106 785 : static std::vector<std::string> GetAliasesStatic()
107 : {
108 : return {
109 : #ifdef GDAL_PIPELINE_PROJ_NOSTALGIA
110 : GDALAlgorithmRegistry::HIDDEN_ALIAS_SEPARATOR,
111 : "+pipeline",
112 : "+gdal=pipeline",
113 : #endif
114 3140 : };
115 : }
116 :
117 : GDALVectorPipelineAlgorithm();
118 :
119 : std::string GetUsageForCLI(bool shortUsage,
120 : const UsageOptions &usageOptions) const override;
121 :
122 : static void RegisterAlgorithms(GDALVectorAlgorithmStepRegistry ®istry,
123 : bool forMixedPipeline);
124 :
125 97 : int GetInputType() const override
126 : {
127 97 : return GDAL_OF_VECTOR;
128 : }
129 :
130 0 : int GetOutputType() const override
131 : {
132 0 : return GDAL_OF_VECTOR;
133 : }
134 :
135 : protected:
136 : GDALVectorAlgorithmStepRegistry m_stepRegistry{};
137 :
138 167 : GDALAlgorithmRegistry &GetStepRegistry() override
139 : {
140 167 : return m_stepRegistry;
141 : }
142 :
143 675 : const GDALAlgorithmRegistry &GetStepRegistry() const override
144 : {
145 675 : return m_stepRegistry;
146 : }
147 :
148 : private:
149 : std::unique_ptr<GDALAbstractPipelineAlgorithm>
150 0 : CreateNestedPipeline() const override
151 : {
152 0 : auto pipeline = std::make_unique<GDALVectorPipelineAlgorithm>();
153 0 : pipeline->m_bInnerPipeline = true;
154 0 : return pipeline;
155 : }
156 : };
157 :
158 : /************************************************************************/
159 : /* GDALVectorOutputDataset */
160 : /************************************************************************/
161 :
162 : class GDALVectorOutputDataset final : public GDALDataset
163 : {
164 :
165 : public:
166 30 : int GetLayerCount() const override
167 : {
168 30 : return static_cast<int>(m_layers.size());
169 : }
170 :
171 16 : const OGRLayer *GetLayer(int idx) const override
172 : {
173 16 : return m_layers[idx].get();
174 : }
175 :
176 : int TestCapability(const char *) const override;
177 :
178 16 : void AddLayer(std::unique_ptr<OGRLayer> layer)
179 : {
180 16 : m_layers.emplace_back(std::move(layer));
181 16 : }
182 :
183 : private:
184 : std::vector<std::unique_ptr<OGRLayer>> m_layers{};
185 : };
186 :
187 : /************************************************************************/
188 : /* GDALVectorAlgorithmLayerProgressHelper */
189 : /************************************************************************/
190 :
191 : /**
192 : * This class helps doing progress report for algorithm iterating over layers
193 : * of the source dataset.
194 : */
195 : class GDALVectorAlgorithmLayerProgressHelper
196 : {
197 : public:
198 : /** Constructor */
199 : GDALVectorAlgorithmLayerProgressHelper(GDALProgressFunc pfnProgress,
200 : void *pProgressData);
201 : /** Constructor */
202 : explicit GDALVectorAlgorithmLayerProgressHelper(
203 : const GDALPipelineStepRunContext &ctxt);
204 :
205 : /** Register the passed layer as a layer that will be processed. */
206 : void AddProcessedLayer(OGRLayer &srcLayer);
207 :
208 : /** Register the passed layer as a layer that will be forwarded without
209 : * processing. */
210 : void AddPassThroughLayer(OGRLayer &srcLayer);
211 :
212 : //! @cond Doxygen_Suppress
213 : class iterator
214 : {
215 : public:
216 96 : explicit iterator(const GDALVectorAlgorithmLayerProgressHelper &helper,
217 : bool start)
218 96 : : m_helper(helper),
219 96 : m_nLayerIdx(start ? 0 : m_helper.m_apoSrcLayers.size())
220 : {
221 96 : }
222 :
223 : inline bool operator==(const iterator &other) const
224 : {
225 : return m_nLayerIdx == other.m_nLayerIdx;
226 : }
227 :
228 90 : inline bool operator!=(const iterator &other) const
229 : {
230 90 : return m_nLayerIdx != other.m_nLayerIdx;
231 : }
232 :
233 42 : inline iterator &operator++()
234 : {
235 42 : if (!m_helper.m_anFeatures.empty())
236 38 : m_nFeatureIdx += m_helper.m_anFeatures[m_nProcessedLayerIdx];
237 42 : if (m_helper.m_apoSrcLayers[m_nLayerIdx].second)
238 36 : ++m_nProcessedLayerIdx;
239 42 : ++m_nLayerIdx;
240 42 : return *this;
241 : }
242 :
243 : using progress_data_unique_ptr =
244 : std::unique_ptr<void, decltype(&GDALDestroyScaledProgress)>;
245 : using value_type = std::tuple<OGRLayer *, bool, GDALProgressFunc,
246 : progress_data_unique_ptr>;
247 :
248 : value_type operator*() const;
249 :
250 : private:
251 : const GDALVectorAlgorithmLayerProgressHelper &m_helper;
252 : size_t m_nLayerIdx = 0;
253 : size_t m_nProcessedLayerIdx = 0;
254 : GIntBig m_nFeatureIdx = 0;
255 : };
256 :
257 : //! @endcond
258 :
259 : /** Start of an iterator over layers registered with AddProcessedLayer()
260 : * and AddUnprocessedLayer() */
261 48 : iterator begin() const
262 : {
263 48 : return iterator(*this, true);
264 : }
265 :
266 : /** End of an iterator over layers registered with AddProcessedLayer()
267 : * and AddUnprocessedLayer() */
268 48 : iterator end() const
269 : {
270 48 : return iterator(*this, false);
271 : }
272 :
273 : /** Return if AddProcessedLayer() has been called at least once. */
274 22 : bool HasProcessedLayers() const
275 : {
276 22 : return !m_anFeatures.empty();
277 : }
278 :
279 : private:
280 : GDALProgressFunc m_pfnProgress = nullptr;
281 : void *m_pProgressData = nullptr;
282 : int64_t m_nTotalFeatures = 0;
283 : std::vector<std::pair<OGRLayer *, bool>> m_apoSrcLayers{};
284 : std::vector<int64_t> m_anFeatures{};
285 :
286 : CPL_DISALLOW_COPY_ASSIGN(GDALVectorAlgorithmLayerProgressHelper)
287 : };
288 :
289 : /************************************************************************/
290 : /* GDALVectorPipelineOutputLayer */
291 : /************************************************************************/
292 :
293 : /** Class that implements GetNextFeature() by forwarding to
294 : * OGRLayerWithTranslateFeature::TranslateFeature() implementation, which
295 : * might return several features.
296 : */
297 186 : class GDALVectorPipelineOutputLayer /* non final */
298 : : public OGRLayerWithTranslateFeature,
299 : public OGRGetNextFeatureThroughRaw<GDALVectorPipelineOutputLayer>
300 : {
301 : protected:
302 : explicit GDALVectorPipelineOutputLayer(OGRLayer &oSrcLayer);
303 : ~GDALVectorPipelineOutputLayer() override;
304 :
305 2121 : DEFINE_GET_NEXT_FEATURE_THROUGH_RAW(GDALVectorPipelineOutputLayer)
306 :
307 : OGRLayer &m_srcLayer;
308 :
309 7 : void FailTranslation()
310 : {
311 7 : m_translateError = true;
312 7 : }
313 :
314 : public:
315 : void ResetReading() override;
316 : OGRFeature *GetNextRawFeature();
317 :
318 : private:
319 : std::vector<std::unique_ptr<OGRFeature>> m_pendingFeatures{};
320 : size_t m_idxInPendingFeatures = 0;
321 : bool m_translateError = false;
322 : };
323 :
324 : /************************************************************************/
325 : /* GDALVectorPipelinePassthroughLayer */
326 : /************************************************************************/
327 :
328 : /** Class that forwards GetNextFeature() calls to the source layer and
329 : * can be added to GDALVectorPipelineOutputDataset::AddLayer()
330 : */
331 : class GDALVectorPipelinePassthroughLayer /* non final */
332 : : public GDALVectorPipelineOutputLayer
333 : {
334 : public:
335 14 : explicit GDALVectorPipelinePassthroughLayer(OGRLayer &oSrcLayer)
336 14 : : GDALVectorPipelineOutputLayer(oSrcLayer)
337 : {
338 14 : }
339 :
340 : const OGRFeatureDefn *GetLayerDefn() const override;
341 :
342 4 : int TestCapability(const char *pszCap) const override
343 : {
344 4 : return m_srcLayer.TestCapability(pszCap);
345 : }
346 :
347 1 : OGRErr IGetExtent(int iGeomField, OGREnvelope *psExtent,
348 : bool bForce) override
349 : {
350 1 : return m_srcLayer.GetExtent(iGeomField, psExtent, bForce);
351 : }
352 :
353 1 : OGRErr IGetExtent3D(int iGeomField, OGREnvelope3D *psExtent,
354 : bool bForce) override
355 : {
356 1 : return m_srcLayer.GetExtent3D(iGeomField, psExtent, bForce);
357 : }
358 :
359 190 : void TranslateFeature(
360 : std::unique_ptr<OGRFeature> poSrcFeature,
361 : std::vector<std::unique_ptr<OGRFeature>> &apoOutFeatures) override
362 : {
363 190 : apoOutFeatures.push_back(std::move(poSrcFeature));
364 190 : }
365 : };
366 :
367 : /************************************************************************/
368 : /* GDALVectorNonStreamingAlgorithmLayer */
369 : /************************************************************************/
370 :
371 : /**
372 : * This class represents a layer for algorithms that process vector data
373 : * in a non-streaming manner.
374 : *
375 : * Implementations must override the following methods:
376 : * - Process(), which is called when the first feature is read
377 : * - GetNextProcessedFeature() method which provides features in sequential order
378 : */
379 42 : class GDALVectorNonStreamingAlgorithmLayer
380 : : public OGRLayer,
381 : public OGRGetNextFeatureThroughRaw<GDALVectorNonStreamingAlgorithmLayer>
382 : {
383 : public:
384 : GDALVectorNonStreamingAlgorithmLayer(OGRLayer &srcLayer,
385 : int geomFieldIndex);
386 :
387 : ~GDALVectorNonStreamingAlgorithmLayer() override;
388 :
389 2 : const char *GetDescription() const override
390 : {
391 2 : return GetName();
392 : }
393 :
394 : virtual bool Process(GDALProgressFunc pfnProgress, void *pProgressData) = 0;
395 :
396 : virtual std::unique_ptr<OGRFeature> GetNextProcessedFeature() = 0;
397 :
398 : OGRFeature *GetNextRawFeature();
399 :
400 1557 : DEFINE_GET_NEXT_FEATURE_THROUGH_RAW(GDALVectorNonStreamingAlgorithmLayer)
401 :
402 : protected:
403 : OGRLayer &m_srcLayer;
404 : int m_geomFieldIndex{0};
405 :
406 : private:
407 : CPL_DISALLOW_COPY_ASSIGN(GDALVectorNonStreamingAlgorithmLayer)
408 : };
409 :
410 : /************************************************************************/
411 : /* GDALVectorNonStreamingAlgorithmDataset */
412 : /************************************************************************/
413 :
414 : /**
415 : * Dataset used to read all input features into memory and perform some
416 : * processing.
417 : */
418 102 : class GDALVectorNonStreamingAlgorithmDataset /* non final */
419 : : public GDALDataset
420 : {
421 : public:
422 : GDALVectorNonStreamingAlgorithmDataset();
423 : ~GDALVectorNonStreamingAlgorithmDataset() override;
424 :
425 : /** Add a layer to the dataset and perform the associated processing. */
426 : bool AddProcessedLayer(
427 : std::unique_ptr<GDALVectorNonStreamingAlgorithmLayer> srcLayer,
428 : GDALProgressFunc progressFunc, void *progressData);
429 :
430 : void AddPassThroughLayer(OGRLayer &oLayer);
431 :
432 : int GetLayerCount() const final override;
433 : OGRLayer *GetLayer(int idx) const final override;
434 : int TestCapability(const char *pszCap) const override;
435 :
436 : private:
437 : std::vector<std::unique_ptr<OGRLayer>> m_owned_layers{};
438 : std::vector<OGRLayer *> m_layers{};
439 : };
440 :
441 : /************************************************************************/
442 : /* GDALVectorPipelineOutputDataset */
443 : /************************************************************************/
444 :
445 : /** Class used by vector pipeline steps to create an output on-the-fly
446 : * dataset where they can store on-the-fly layers.
447 : */
448 : class GDALVectorPipelineOutputDataset final : public GDALDataset
449 : {
450 : GDALDataset &m_srcDS;
451 : std::map<OGRLayer *, OGRLayerWithTranslateFeature *>
452 : m_mapSrcLayerToNewLayer{};
453 : std::vector<std::unique_ptr<OGRLayerWithTranslateFeature>>
454 : m_layersToDestroy{};
455 : std::vector<OGRLayerWithTranslateFeature *> m_layers{};
456 :
457 : OGRLayerWithTranslateFeature *m_belongingLayer = nullptr;
458 : std::vector<std::unique_ptr<OGRFeature>> m_pendingFeatures{};
459 : size_t m_idxInPendingFeatures = 0;
460 :
461 : CPL_DISALLOW_COPY_ASSIGN(GDALVectorPipelineOutputDataset)
462 :
463 : public:
464 : explicit GDALVectorPipelineOutputDataset(GDALDataset &oSrcDS);
465 :
466 : void AddLayer(OGRLayer &oSrcLayer,
467 : std::unique_ptr<OGRLayerWithTranslateFeature> poNewLayer);
468 :
469 : int GetLayerCount() const override;
470 :
471 : OGRLayer *GetLayer(int idx) const override;
472 :
473 : int TestCapability(const char *pszCap) const override;
474 :
475 : void ResetReading() override;
476 :
477 : OGRFeature *GetNextFeature(OGRLayer **ppoBelongingLayer,
478 : double *pdfProgressPct,
479 : GDALProgressFunc pfnProgress,
480 : void *pProgressData) override;
481 : };
482 :
483 : //! @endcond
484 :
485 : #endif
|
