Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: GDAL
4 : * Purpose: gdal "vector pipeline" subcommand
5 : * Author: Even Rouault <even dot rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2024, Even Rouault <even dot rouault at spatialys.com>
9 : *
10 : * SPDX-License-Identifier: MIT
11 : ****************************************************************************/
12 :
13 : #ifndef GDALALG_VECTOR_PIPELINE_INCLUDED
14 : #define GDALALG_VECTOR_PIPELINE_INCLUDED
15 :
16 : #include "gdalalgorithm.h"
17 : #include "gdalalg_abstract_pipeline.h"
18 :
19 : #include "ogrsf_frmts.h"
20 : #include "ogrlayerwithtranslatefeature.h"
21 :
22 : #include <map>
23 : #include <tuple>
24 : #include <vector>
25 :
26 : //! @cond Doxygen_Suppress
27 :
28 : /************************************************************************/
29 : /* GDALVectorPipelineStepAlgorithm */
30 : /************************************************************************/
31 :
32 : class GDALRasterAlgorithmStepRegistry;
33 :
34 2488 : class GDALVectorPipelineStepAlgorithm /* non final */
35 : : public GDALPipelineStepAlgorithm
36 : {
37 : public:
38 : ~GDALVectorPipelineStepAlgorithm() override;
39 :
40 : protected:
41 : GDALVectorPipelineStepAlgorithm(const std::string &name,
42 : const std::string &description,
43 : const std::string &helpURL,
44 : bool standaloneStep);
45 :
46 : GDALVectorPipelineStepAlgorithm(const std::string &name,
47 : const std::string &description,
48 : const std::string &helpURL,
49 : const ConstructorOptions &options);
50 :
51 : friend class GDALVectorPipelineAlgorithm;
52 : friend class GDALVectorConcatAlgorithm;
53 :
54 358 : int GetInputType() const override
55 : {
56 358 : return GDAL_OF_VECTOR;
57 : }
58 :
59 342 : int GetOutputType() const override
60 : {
61 342 : return GDAL_OF_VECTOR;
62 : }
63 : };
64 :
65 : /************************************************************************/
66 : /* GDALVectorAlgorithmStepRegistry */
67 : /************************************************************************/
68 :
69 307 : class GDALVectorAlgorithmStepRegistry : public virtual GDALAlgorithmRegistry
70 : {
71 : public:
72 307 : GDALVectorAlgorithmStepRegistry() = default;
73 : ~GDALVectorAlgorithmStepRegistry() override;
74 :
75 : /** Register the algorithm of type MyAlgorithm.
76 : */
77 : template <class MyAlgorithm>
78 8903 : bool Register(const std::string &name = std::string())
79 : {
80 : static_assert(
81 : std::is_base_of_v<GDALVectorPipelineStepAlgorithm, MyAlgorithm>,
82 : "Algorithm is not a GDALVectorPipelineStepAlgorithm");
83 :
84 17806 : AlgInfo info;
85 8903 : info.m_name = name.empty() ? MyAlgorithm::NAME : name;
86 8903 : info.m_aliases = MyAlgorithm::GetAliasesStatic();
87 10183 : info.m_creationFunc = []() -> std::unique_ptr<GDALAlgorithm>
88 1280 : { return std::make_unique<MyAlgorithm>(); };
89 17806 : return GDALAlgorithmRegistry::Register(info);
90 : }
91 : };
92 :
93 : /************************************************************************/
94 : /* GDALVectorPipelineAlgorithm */
95 : /************************************************************************/
96 :
97 : class GDALVectorPipelineAlgorithm final : public GDALAbstractPipelineAlgorithm
98 : {
99 : public:
100 : static constexpr const char *NAME = "pipeline";
101 : static constexpr const char *DESCRIPTION =
102 : "Process a vector dataset applying several steps.";
103 : static constexpr const char *HELP_URL =
104 : "/programs/gdal_vector_pipeline.html";
105 :
106 753 : static std::vector<std::string> GetAliasesStatic()
107 : {
108 : return {
109 : #ifdef GDAL_PIPELINE_PROJ_NOSTALGIA
110 : GDALAlgorithmRegistry::HIDDEN_ALIAS_SEPARATOR,
111 : "+pipeline",
112 : "+gdal=pipeline",
113 : #endif
114 3012 : };
115 : }
116 :
117 : GDALVectorPipelineAlgorithm();
118 :
119 : std::string GetUsageForCLI(bool shortUsage,
120 : const UsageOptions &usageOptions) const override;
121 :
122 : static void RegisterAlgorithms(GDALVectorAlgorithmStepRegistry ®istry,
123 : bool forMixedPipeline);
124 :
125 96 : int GetInputType() const override
126 : {
127 96 : return GDAL_OF_VECTOR;
128 : }
129 :
130 0 : int GetOutputType() const override
131 : {
132 0 : return GDAL_OF_VECTOR;
133 : }
134 :
135 : protected:
136 : GDALVectorAlgorithmStepRegistry m_stepRegistry{};
137 :
138 167 : GDALAlgorithmRegistry &GetStepRegistry() override
139 : {
140 167 : return m_stepRegistry;
141 : }
142 :
143 672 : const GDALAlgorithmRegistry &GetStepRegistry() const override
144 : {
145 672 : return m_stepRegistry;
146 : }
147 :
148 : private:
149 : std::unique_ptr<GDALAbstractPipelineAlgorithm>
150 0 : CreateNestedPipeline() const override
151 : {
152 0 : auto pipeline = std::make_unique<GDALVectorPipelineAlgorithm>();
153 0 : pipeline->m_bInnerPipeline = true;
154 0 : return pipeline;
155 : }
156 : };
157 :
158 : /************************************************************************/
159 : /* GDALVectorOutputDataset */
160 : /************************************************************************/
161 :
162 : class GDALVectorOutputDataset final : public GDALDataset
163 : {
164 :
165 : public:
166 30 : int GetLayerCount() const override
167 : {
168 30 : return static_cast<int>(m_layers.size());
169 : }
170 :
171 16 : const OGRLayer *GetLayer(int idx) const override
172 : {
173 16 : return m_layers[idx].get();
174 : }
175 :
176 : int TestCapability(const char *) const override;
177 :
178 16 : void AddLayer(std::unique_ptr<OGRLayer> layer)
179 : {
180 16 : m_layers.emplace_back(std::move(layer));
181 16 : }
182 :
183 : private:
184 : std::vector<std::unique_ptr<OGRLayer>> m_layers{};
185 : };
186 :
187 : /************************************************************************/
188 : /* GDALVectorAlgorithmLayerProgressHelper */
189 : /************************************************************************/
190 :
191 : /**
192 : * This class helps doing progress report for algorithm iterating over layers
193 : * of the source dataset.
194 : */
195 : class GDALVectorAlgorithmLayerProgressHelper
196 : {
197 : public:
198 : /** Constructor */
199 : GDALVectorAlgorithmLayerProgressHelper(GDALProgressFunc pfnProgress,
200 : void *pProgressData);
201 : /** Constructor */
202 : explicit GDALVectorAlgorithmLayerProgressHelper(
203 : const GDALPipelineStepRunContext &ctxt);
204 :
205 : /** Register the passed layer as a layer that will be processed. */
206 : void AddProcessedLayer(OGRLayer &srcLayer);
207 :
208 : /** Register the passed layer as a layer that will be forwarded without
209 : * processing. */
210 : void AddPassThroughLayer(OGRLayer &srcLayer);
211 :
212 : //! @cond Doxygen_Suppress
213 : class iterator
214 : {
215 : public:
216 80 : explicit iterator(const GDALVectorAlgorithmLayerProgressHelper &helper,
217 : bool start)
218 80 : : m_helper(helper),
219 80 : m_nLayerIdx(start ? 0 : m_helper.m_apoSrcLayers.size())
220 : {
221 80 : }
222 :
223 : inline bool operator==(const iterator &other) const
224 : {
225 : return m_nLayerIdx == other.m_nLayerIdx;
226 : }
227 :
228 74 : inline bool operator!=(const iterator &other) const
229 : {
230 74 : return m_nLayerIdx != other.m_nLayerIdx;
231 : }
232 :
233 34 : inline iterator &operator++()
234 : {
235 34 : if (!m_helper.m_anFeatures.empty())
236 32 : m_nFeatureIdx += m_helper.m_anFeatures[m_nProcessedLayerIdx];
237 34 : if (m_helper.m_apoSrcLayers[m_nLayerIdx].second)
238 30 : ++m_nProcessedLayerIdx;
239 34 : ++m_nLayerIdx;
240 34 : return *this;
241 : }
242 :
243 : using progress_data_unique_ptr =
244 : std::unique_ptr<void, decltype(&GDALDestroyScaledProgress)>;
245 : using value_type = std::tuple<OGRLayer *, bool, GDALProgressFunc,
246 : progress_data_unique_ptr>;
247 :
248 : value_type operator*() const;
249 :
250 : private:
251 : const GDALVectorAlgorithmLayerProgressHelper &m_helper;
252 : size_t m_nLayerIdx = 0;
253 : size_t m_nProcessedLayerIdx = 0;
254 : GIntBig m_nFeatureIdx = 0;
255 : };
256 :
257 : //! @endcond
258 :
259 : /** Start of an iterator over layers registered with AddProcessedLayer()
260 : * and AddUnprocessedLayer() */
261 40 : iterator begin() const
262 : {
263 40 : return iterator(*this, true);
264 : }
265 :
266 : /** End of an iterator over layers registered with AddProcessedLayer()
267 : * and AddUnprocessedLayer() */
268 40 : iterator end() const
269 : {
270 40 : return iterator(*this, false);
271 : }
272 :
273 : /** Return if AddProcessedLayer() has been called at least once. */
274 22 : bool HasProcessedLayers() const
275 : {
276 22 : return !m_anFeatures.empty();
277 : }
278 :
279 : private:
280 : GDALProgressFunc m_pfnProgress = nullptr;
281 : void *m_pProgressData = nullptr;
282 : int64_t m_nTotalFeatures = 0;
283 : std::vector<std::pair<OGRLayer *, bool>> m_apoSrcLayers{};
284 : std::vector<int64_t> m_anFeatures{};
285 :
286 : CPL_DISALLOW_COPY_ASSIGN(GDALVectorAlgorithmLayerProgressHelper)
287 : };
288 :
289 : /************************************************************************/
290 : /* GDALVectorPipelineOutputLayer */
291 : /************************************************************************/
292 :
293 : /** Class that implements GetNextFeature() by forwarding to
294 : * OGRLayerWithTranslateFeature::TranslateFeature() implementation, which
295 : * might return several features.
296 : */
297 183 : class GDALVectorPipelineOutputLayer /* non final */
298 : : public OGRLayerWithTranslateFeature,
299 : public OGRGetNextFeatureThroughRaw<GDALVectorPipelineOutputLayer>
300 : {
301 : protected:
302 : explicit GDALVectorPipelineOutputLayer(OGRLayer &oSrcLayer);
303 : ~GDALVectorPipelineOutputLayer() override;
304 :
305 2020 : DEFINE_GET_NEXT_FEATURE_THROUGH_RAW(GDALVectorPipelineOutputLayer)
306 :
307 : OGRLayer &m_srcLayer;
308 :
309 7 : void FailTranslation()
310 : {
311 7 : m_translateError = true;
312 7 : }
313 :
314 : public:
315 : void ResetReading() override;
316 : OGRFeature *GetNextRawFeature();
317 :
318 : private:
319 : std::vector<std::unique_ptr<OGRFeature>> m_pendingFeatures{};
320 : size_t m_idxInPendingFeatures = 0;
321 : bool m_translateError = false;
322 : };
323 :
324 : /************************************************************************/
325 : /* GDALVectorPipelinePassthroughLayer */
326 : /************************************************************************/
327 :
328 : /** Class that forwards GetNextFeature() calls to the source layer and
329 : * can be added to GDALVectorPipelineOutputDataset::AddLayer()
330 : */
331 : class GDALVectorPipelinePassthroughLayer /* non final */
332 : : public GDALVectorPipelineOutputLayer
333 : {
334 : public:
335 12 : explicit GDALVectorPipelinePassthroughLayer(OGRLayer &oSrcLayer)
336 12 : : GDALVectorPipelineOutputLayer(oSrcLayer)
337 : {
338 12 : }
339 :
340 : const OGRFeatureDefn *GetLayerDefn() const override;
341 :
342 4 : int TestCapability(const char *pszCap) const override
343 : {
344 4 : return m_srcLayer.TestCapability(pszCap);
345 : }
346 :
347 1 : OGRErr IGetExtent(int iGeomField, OGREnvelope *psExtent,
348 : bool bForce) override
349 : {
350 1 : return m_srcLayer.GetExtent(iGeomField, psExtent, bForce);
351 : }
352 :
353 1 : OGRErr IGetExtent3D(int iGeomField, OGREnvelope3D *psExtent,
354 : bool bForce) override
355 : {
356 1 : return m_srcLayer.GetExtent3D(iGeomField, psExtent, bForce);
357 : }
358 :
359 110 : void TranslateFeature(
360 : std::unique_ptr<OGRFeature> poSrcFeature,
361 : std::vector<std::unique_ptr<OGRFeature>> &apoOutFeatures) override
362 : {
363 110 : apoOutFeatures.push_back(std::move(poSrcFeature));
364 110 : }
365 : };
366 :
367 : /************************************************************************/
368 : /* GDALVectorNonStreamingAlgorithmDataset */
369 : /************************************************************************/
370 :
371 : class MEMDataset;
372 :
373 : /**
374 : * Dataset used to read all input features into memory and perform some
375 : * processing.
376 : */
377 43 : class GDALVectorNonStreamingAlgorithmDataset /* non final */
378 : : public GDALDataset
379 : {
380 : public:
381 : GDALVectorNonStreamingAlgorithmDataset();
382 : ~GDALVectorNonStreamingAlgorithmDataset() override;
383 :
384 : virtual bool Process(OGRLayer &srcLayer, OGRLayer &dstLayer,
385 : int geomFieldIndex, GDALProgressFunc pfnProgress,
386 : void *pProgressData) = 0;
387 :
388 : bool AddProcessedLayer(OGRLayer &srcLayer, GDALProgressFunc pfnProgress,
389 : void *pProgressData);
390 : bool AddProcessedLayer(OGRLayer &srcLayer, OGRFeatureDefn &dstDefn,
391 : int geomFieldIndex, GDALProgressFunc pfnProgress,
392 : void *pProgressData);
393 : void AddPassThroughLayer(OGRLayer &oLayer);
394 : int GetLayerCount() const final override;
395 : OGRLayer *GetLayer(int idx) const final override;
396 : int TestCapability(const char *pszCap) const override;
397 :
398 : private:
399 : std::vector<std::unique_ptr<OGRLayer>> m_passthrough_layers{};
400 : std::vector<OGRLayer *> m_layers{};
401 : std::unique_ptr<MEMDataset> m_ds{};
402 : };
403 :
404 : /************************************************************************/
405 : /* GDALVectorPipelineOutputDataset */
406 : /************************************************************************/
407 :
408 : /** Class used by vector pipeline steps to create an output on-the-fly
409 : * dataset where they can store on-the-fly layers.
410 : */
411 : class GDALVectorPipelineOutputDataset final : public GDALDataset
412 : {
413 : GDALDataset &m_srcDS;
414 : std::map<OGRLayer *, OGRLayerWithTranslateFeature *>
415 : m_mapSrcLayerToNewLayer{};
416 : std::vector<std::unique_ptr<OGRLayerWithTranslateFeature>>
417 : m_layersToDestroy{};
418 : std::vector<OGRLayerWithTranslateFeature *> m_layers{};
419 :
420 : OGRLayerWithTranslateFeature *m_belongingLayer = nullptr;
421 : std::vector<std::unique_ptr<OGRFeature>> m_pendingFeatures{};
422 : size_t m_idxInPendingFeatures = 0;
423 :
424 : CPL_DISALLOW_COPY_ASSIGN(GDALVectorPipelineOutputDataset)
425 :
426 : public:
427 : explicit GDALVectorPipelineOutputDataset(GDALDataset &oSrcDS);
428 :
429 : void AddLayer(OGRLayer &oSrcLayer,
430 : std::unique_ptr<OGRLayerWithTranslateFeature> poNewLayer);
431 :
432 : int GetLayerCount() const override;
433 :
434 : OGRLayer *GetLayer(int idx) const override;
435 :
436 : int TestCapability(const char *pszCap) const override;
437 :
438 : void ResetReading() override;
439 :
440 : OGRFeature *GetNextFeature(OGRLayer **ppoBelongingLayer,
441 : double *pdfProgressPct,
442 : GDALProgressFunc pfnProgress,
443 : void *pProgressData) override;
444 : };
445 :
446 : //! @endcond
447 :
448 : #endif
|
