Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: GDAL
4 : * Purpose: gdal "pipeline" subcommand
5 : * Author: Even Rouault <even dot rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2024, Even Rouault <even dot rouault at spatialys.com>
9 : *
10 : * SPDX-License-Identifier: MIT
11 : ****************************************************************************/
12 :
13 : //! @cond Doxygen_Suppress
14 :
15 : #include "gdalalg_pipeline.h"
16 : #include "cpl_error.h"
17 : #include "gdal_priv.h"
18 :
19 : #include "gdalalg_external.h"
20 :
21 : #include "gdalalg_raster_read.h"
22 : #include "gdalalg_raster_mosaic.h"
23 : #include "gdalalg_raster_stack.h"
24 : #include "gdalalg_raster_write.h"
25 : #include "gdalalg_raster_zonal_stats.h"
26 :
27 : #include "gdalalg_vector_read.h"
28 : #include "gdalalg_vector_write.h"
29 :
30 : #include "gdalalg_raster_as_features.h"
31 : #include "gdalalg_raster_compare.h"
32 : #include "gdalalg_raster_contour.h"
33 : #include "gdalalg_raster_footprint.h"
34 : #include "gdalalg_raster_polygonize.h"
35 : #include "gdalalg_raster_info.h"
36 : #include "gdalalg_raster_pixel_info.h"
37 : #include "gdalalg_raster_tile.h"
38 : #include "gdalalg_vector_grid.h"
39 : #include "gdalalg_vector_info.h"
40 : #include "gdalalg_vector_rasterize.h"
41 :
42 : #include <algorithm>
43 : #include <cassert>
44 :
45 : #ifndef _
46 : #define _(x) (x)
47 : #endif
48 :
49 : /************************************************************************/
50 : /* GDALPipelineStepAlgorithm() */
51 : /************************************************************************/
52 :
53 14045 : GDALPipelineStepAlgorithm::GDALPipelineStepAlgorithm(
54 : const std::string &name, const std::string &description,
55 14045 : const std::string &helpURL, const ConstructorOptions &options)
56 : : GDALAlgorithm(name, description, helpURL),
57 14045 : m_standaloneStep(options.standaloneStep), m_constructorOptions(options)
58 : {
59 14045 : }
60 :
61 : /************************************************************************/
62 : /* GDALPipelineStepAlgorithm::AddRasterHiddenInputDatasetArg() */
63 : /************************************************************************/
64 :
65 2153 : void GDALPipelineStepAlgorithm::AddRasterHiddenInputDatasetArg()
66 : {
67 2153 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_RASTER, false)
68 2153 : .SetMinCount(0)
69 2153 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
70 2153 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
71 2153 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
72 2153 : .SetHidden();
73 2153 : }
74 :
75 : /************************************************************************/
76 : /* GDALPipelineStepAlgorithm::AddRasterInputArgs() */
77 : /************************************************************************/
78 :
79 4717 : void GDALPipelineStepAlgorithm::AddRasterInputArgs(
80 : bool openForMixedRasterVector, bool hiddenForCLI)
81 : {
82 4717 : AddInputFormatsArg(&m_inputFormats)
83 : .AddMetadataItem(
84 : GAAMDI_REQUIRED_CAPABILITIES,
85 : openForMixedRasterVector
86 14489 : ? std::vector<std::string>{GDAL_DCAP_RASTER, GDAL_DCAP_VECTOR}
87 13813 : : std::vector<std::string>{GDAL_DCAP_RASTER})
88 4717 : .SetHiddenForCLI(hiddenForCLI)
89 4717 : .SetAvailableInPipelineStep(false);
90 4717 : AddOpenOptionsArg(&m_openOptions)
91 4717 : .SetHiddenForCLI(hiddenForCLI)
92 4717 : .SetAvailableInPipelineStep(false);
93 : auto &arg =
94 : AddInputDatasetArg(
95 : &m_inputDataset,
96 : openForMixedRasterVector ? (GDAL_OF_RASTER | GDAL_OF_VECTOR)
97 : : GDAL_OF_RASTER,
98 4717 : false, m_constructorOptions.inputDatasetHelpMsg.c_str())
99 4717 : .SetDatasetInputFlags(m_constructorOptions.inputDatasetInputFlags)
100 4717 : .SetMinCount(m_constructorOptions.inputDatasetRequired ? 1 : 0)
101 4717 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
102 4717 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
103 4717 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
104 4717 : .SetHiddenForCLI(hiddenForCLI)
105 4717 : .SetAvailableInPipelineStep(false);
106 4717 : if (m_constructorOptions.inputDatasetPositional && !hiddenForCLI)
107 4386 : arg.SetPositional();
108 4717 : if (m_constructorOptions.inputDatasetRequired && !hiddenForCLI)
109 4386 : arg.SetRequired();
110 4717 : if (!m_constructorOptions.inputDatasetAlias.empty())
111 841 : arg.AddAlias(m_constructorOptions.inputDatasetAlias);
112 4717 : }
113 :
114 : /************************************************************************/
115 : /* GDALPipelineStepAlgorithm::AddRasterOutputArgs() */
116 : /************************************************************************/
117 :
118 3062 : void GDALPipelineStepAlgorithm::AddRasterOutputArgs(bool hiddenForCLI)
119 : {
120 3062 : m_outputFormatArg =
121 : &(AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
122 3062 : /* bGDALGAllowed = */ true)
123 : .AddMetadataItem(
124 : GAAMDI_REQUIRED_CAPABILITIES,
125 : {GDAL_DCAP_RASTER,
126 12248 : m_constructorOptions.outputFormatCreateCapability.c_str()})
127 3062 : .SetHiddenForCLI(hiddenForCLI))
128 3062 : .SetAvailableInPipelineStep(false);
129 : AddOutputDatasetArg(&m_outputDataset, GDAL_OF_RASTER,
130 3062 : /* positionalAndRequired = */ !hiddenForCLI,
131 3062 : m_constructorOptions.outputDatasetHelpMsg.c_str())
132 3062 : .SetHiddenForCLI(hiddenForCLI)
133 3062 : .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT)
134 3062 : .SetAvailableInPipelineStep(false);
135 3062 : AddCreationOptionsArg(&m_creationOptions)
136 3062 : .SetHiddenForCLI(hiddenForCLI)
137 3062 : .SetAvailableInPipelineStep(false);
138 3062 : constexpr const char *MUTUAL_EXCLUSION_GROUP_OVERWRITE_APPEND =
139 : "overwrite-append";
140 3062 : AddOverwriteArg(&m_overwrite)
141 3062 : .SetHiddenForCLI(hiddenForCLI)
142 6124 : .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_OVERWRITE_APPEND)
143 3062 : .SetAvailableInPipelineStep(false);
144 : AddArg(GDAL_ARG_NAME_APPEND, 0,
145 6124 : _("Append as a subdataset to existing output"), &m_appendRaster)
146 3062 : .SetDefault(false)
147 3062 : .SetHiddenForCLI(hiddenForCLI)
148 6124 : .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_OVERWRITE_APPEND)
149 3062 : .SetAvailableInPipelineStep(false);
150 3062 : }
151 :
152 : /************************************************************************/
153 : /* GDALPipelineStepAlgorithm::AddVectorHiddenInputDatasetArg() */
154 : /************************************************************************/
155 :
156 2006 : void GDALPipelineStepAlgorithm::AddVectorHiddenInputDatasetArg()
157 : {
158 2006 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_VECTOR, false)
159 2006 : .SetMinCount(0)
160 2006 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
161 2006 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
162 2006 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
163 2006 : .SetHidden();
164 2006 : }
165 :
166 : /************************************************************************/
167 : /* GDALPipelineStepAlgorithm::AddVectorInputArgs() */
168 : /************************************************************************/
169 :
170 2745 : void GDALPipelineStepAlgorithm::AddVectorInputArgs(bool hiddenForCLI)
171 : {
172 2745 : AddInputFormatsArg(&m_inputFormats)
173 8235 : .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES, {GDAL_DCAP_VECTOR})
174 2745 : .SetHiddenForCLI(hiddenForCLI)
175 2745 : .SetAvailableInPipelineStep(false);
176 2745 : AddOpenOptionsArg(&m_openOptions)
177 2745 : .SetHiddenForCLI(hiddenForCLI)
178 2745 : .SetAvailableInPipelineStep(false);
179 : auto &datasetArg =
180 2745 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_VECTOR, false)
181 2745 : .SetMinCount(m_constructorOptions.inputDatasetRequired ? 1 : 0)
182 2745 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
183 2745 : .SetDatasetInputFlags(m_constructorOptions.inputDatasetInputFlags)
184 2745 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
185 2745 : .SetHiddenForCLI(hiddenForCLI)
186 2745 : .SetAvailableInPipelineStep(false);
187 2745 : if (!m_constructorOptions.inputDatasetAlias.empty())
188 190 : datasetArg.AddAlias(m_constructorOptions.inputDatasetAlias);
189 2745 : if (!m_constructorOptions.inputDatasetMetaVar.empty())
190 2745 : datasetArg.SetMetaVar(m_constructorOptions.inputDatasetMetaVar);
191 2745 : if (m_constructorOptions.inputDatasetPositional && !hiddenForCLI)
192 2446 : datasetArg.SetPositional();
193 2745 : if (m_constructorOptions.inputDatasetRequired && !hiddenForCLI)
194 2446 : datasetArg.SetRequired();
195 2745 : if (m_constructorOptions.addInputLayerNameArgument)
196 : {
197 : auto &layerArg = AddArg(GDAL_ARG_NAME_INPUT_LAYER, 'l',
198 4950 : _("Input layer name(s)"), &m_inputLayerNames)
199 4950 : .AddAlias("layer")
200 2475 : .SetHiddenForCLI(hiddenForCLI)
201 2475 : .SetAvailableInPipelineStep(false);
202 2475 : SetAutoCompleteFunctionForLayerName(layerArg, datasetArg);
203 : }
204 2745 : }
205 :
206 : /************************************************************************/
207 : /* GDALPipelineStepAlgorithm::AddVectorOutputArgs() */
208 : /************************************************************************/
209 :
210 3157 : void GDALPipelineStepAlgorithm::AddVectorOutputArgs(
211 : bool hiddenForCLI, bool shortNameOutputLayerAllowed)
212 : {
213 : AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
214 3157 : /* bGDALGAllowed = */ true)
215 : .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,
216 12628 : {GDAL_DCAP_VECTOR, GDAL_DCAP_CREATE})
217 3157 : .SetHiddenForCLI(hiddenForCLI)
218 3157 : .SetAvailableInPipelineStep(false);
219 3157 : AddOutputOpenOptionsArg(&m_outputOpenOptions)
220 3157 : .SetHiddenForCLI(hiddenForCLI)
221 3157 : .SetAvailableInPipelineStep(false);
222 : auto &outputDatasetArg =
223 : AddOutputDatasetArg(&m_outputDataset, GDAL_OF_VECTOR,
224 3157 : /* positionalAndRequired = */ false)
225 3157 : .SetHiddenForCLI(hiddenForCLI)
226 3157 : .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT)
227 3157 : .SetAvailableInPipelineStep(false);
228 3157 : if (!hiddenForCLI)
229 2989 : outputDatasetArg.SetPositional();
230 3157 : if (!hiddenForCLI && m_constructorOptions.outputDatasetRequired)
231 2930 : outputDatasetArg.SetRequired();
232 :
233 3157 : AddCreationOptionsArg(&m_creationOptions)
234 3157 : .SetHiddenForCLI(hiddenForCLI)
235 3157 : .SetAvailableInPipelineStep(false);
236 3157 : AddLayerCreationOptionsArg(&m_layerCreationOptions)
237 3157 : .SetHiddenForCLI(hiddenForCLI)
238 3157 : .SetAvailableInPipelineStep(false);
239 3157 : AddOverwriteArg(&m_overwrite)
240 3157 : .SetHiddenForCLI(hiddenForCLI)
241 3157 : .SetAvailableInPipelineStep(false);
242 3157 : GDALInConstructionAlgorithmArg *updateArg = nullptr;
243 3157 : if (m_constructorOptions.addUpdateArgument)
244 : {
245 3095 : updateArg = &AddUpdateArg(&m_update)
246 3095 : .SetHiddenForCLI(hiddenForCLI)
247 3095 : .SetAvailableInPipelineStep(false);
248 : }
249 3157 : if (m_constructorOptions.addOverwriteLayerArgument)
250 : {
251 3095 : AddOverwriteLayerArg(&m_overwriteLayer)
252 3095 : .SetHiddenForCLI(hiddenForCLI)
253 3095 : .SetAvailableInPipelineStep(false);
254 : }
255 3157 : constexpr const char *MUTUAL_EXCLUSION_GROUP_APPEND_UPSERT =
256 : "append-upsert";
257 3157 : if (m_constructorOptions.addAppendLayerArgument)
258 : {
259 2964 : AddAppendLayerArg(&m_appendLayer)
260 2964 : .SetHiddenForCLI(hiddenForCLI)
261 5928 : .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_APPEND_UPSERT)
262 2964 : .SetAvailableInPipelineStep(false);
263 : }
264 3157 : if (m_constructorOptions.addUpsertArgument)
265 : {
266 5670 : AddArg("upsert", 0, _("Upsert features (implies 'append')"), &m_upsert)
267 2835 : .SetHiddenForCLI(hiddenForCLI)
268 5670 : .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_APPEND_UPSERT)
269 :
270 2835 : .SetAvailableInPipelineStep(false)
271 : .AddAction(
272 12 : [updateArg, this]()
273 : {
274 4 : if (m_upsert && updateArg)
275 4 : updateArg->Set(true);
276 5670 : })
277 2835 : .SetCategory(GAAC_ADVANCED);
278 : }
279 3157 : if (m_constructorOptions.addOutputLayerNameArgument)
280 : {
281 3029 : AddOutputLayerNameArg(hiddenForCLI, shortNameOutputLayerAllowed);
282 : }
283 3157 : if (m_constructorOptions.addSkipErrorsArgument)
284 : {
285 : AddArg("skip-errors", 0, _("Skip errors when writing features"),
286 5670 : &m_skipErrors)
287 5670 : .AddHiddenAlias("skip-failures") // For ogr2ogr nostalgic people
288 2835 : .SetAvailableInPipelineStep(false);
289 : }
290 3157 : if (m_constructorOptions.addNoCreateEmptyLayersArgument)
291 : {
292 : AddArg("no-create-empty-layers", 0,
293 : _("Avoid creating layers to which no features will be written"),
294 2156 : &m_noCreateEmptyLayers)
295 1078 : .SetAvailableInPipelineStep(false);
296 : }
297 3157 : }
298 :
299 : /************************************************************************/
300 : /* GDALPipelineStepAlgorithm::AddOutputLayerNameArg() */
301 : /************************************************************************/
302 :
303 3226 : void GDALPipelineStepAlgorithm::AddOutputLayerNameArg(
304 : bool hiddenForCLI, bool shortNameOutputLayerAllowed)
305 : {
306 : AddArg(GDAL_ARG_NAME_OUTPUT_LAYER, shortNameOutputLayerAllowed ? 'l' : 0,
307 : _("Output layer name"),
308 6452 : &m_outputLayerName)
309 6452 : .AddHiddenAlias("nln") // For ogr2ogr nostalgic people
310 3226 : .SetHiddenForCLI(hiddenForCLI)
311 : .SetAvailableInPipelineStep(
312 3226 : m_constructorOptions.outputLayerNameAvailableInPipelineStep);
313 3226 : }
314 :
315 : /************************************************************************/
316 : /* GDALPipelineStepAlgorithm::AddMdimHiddenInputDatasetArg() */
317 : /************************************************************************/
318 :
319 8 : void GDALPipelineStepAlgorithm::AddMdimHiddenInputDatasetArg()
320 : {
321 8 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_MULTIDIM_RASTER, false)
322 8 : .SetMinCount(0)
323 8 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
324 8 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
325 8 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
326 8 : .SetHidden();
327 8 : }
328 :
329 : /************************************************************************/
330 : /* GDALPipelineStepAlgorithm::AddMdimInputArgs() */
331 : /************************************************************************/
332 :
333 205 : void GDALPipelineStepAlgorithm::AddMdimInputArgs(bool openForMixedMdimVector,
334 : bool hiddenForCLI,
335 : bool acceptRaster)
336 : {
337 205 : AddInputFormatsArg(&m_inputFormats)
338 : .AddMetadataItem(
339 : GAAMDI_REQUIRED_CAPABILITIES,
340 : openForMixedMdimVector
341 615 : ? std::vector<
342 : std::string>{acceptRaster
343 : ? GDAL_ALG_DCAP_RASTER_OR_MULTIDIM_RASTER
344 : : GDAL_DCAP_MULTIDIM_RASTER,
345 0 : GDAL_DCAP_VECTOR}
346 : : std::vector<
347 : std::string>{acceptRaster
348 : ? GDAL_ALG_DCAP_RASTER_OR_MULTIDIM_RASTER
349 615 : : GDAL_DCAP_MULTIDIM_RASTER})
350 205 : .SetHiddenForCLI(hiddenForCLI)
351 205 : .SetAvailableInPipelineStep(false);
352 205 : AddOpenOptionsArg(&m_openOptions)
353 205 : .SetHiddenForCLI(hiddenForCLI)
354 205 : .SetAvailableInPipelineStep(false);
355 : auto &arg =
356 : AddInputDatasetArg(&m_inputDataset,
357 205 : (acceptRaster ? GDAL_OF_RASTER : 0) |
358 : (openForMixedMdimVector
359 205 : ? (GDAL_OF_MULTIDIM_RASTER | GDAL_OF_VECTOR)
360 : : GDAL_OF_MULTIDIM_RASTER),
361 : false,
362 410 : m_constructorOptions.inputDatasetHelpMsg.c_str())
363 205 : .SetDatasetInputFlags(m_constructorOptions.inputDatasetInputFlags)
364 205 : .SetMinCount(m_constructorOptions.inputDatasetRequired ? 1 : 0)
365 205 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
366 205 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
367 205 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
368 205 : .SetHiddenForCLI(hiddenForCLI)
369 205 : .SetAvailableInPipelineStep(false);
370 205 : if (m_constructorOptions.inputDatasetPositional && !hiddenForCLI)
371 156 : arg.SetPositional();
372 205 : if (m_constructorOptions.inputDatasetRequired && !hiddenForCLI)
373 156 : arg.SetRequired();
374 205 : if (!m_constructorOptions.inputDatasetAlias.empty())
375 50 : arg.AddAlias(m_constructorOptions.inputDatasetAlias);
376 205 : }
377 :
378 : /************************************************************************/
379 : /* GDALPipelineStepAlgorithm::AddMdimOutputArgs() */
380 : /************************************************************************/
381 :
382 154 : void GDALPipelineStepAlgorithm::AddMdimOutputArgs(bool hiddenForCLI)
383 : {
384 154 : m_outputFormatArg =
385 : &(AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
386 154 : /* bGDALGAllowed = */ true)
387 : .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,
388 462 : {GDAL_DCAP_CREATE_MULTIDIMENSIONAL})
389 154 : .SetHiddenForCLI(hiddenForCLI))
390 154 : .SetAvailableInPipelineStep(false);
391 : AddOutputDatasetArg(&m_outputDataset, GDAL_OF_MULTIDIM_RASTER,
392 154 : /* positionalAndRequired = */ !hiddenForCLI,
393 154 : m_constructorOptions.outputDatasetHelpMsg.c_str())
394 154 : .SetHiddenForCLI(hiddenForCLI)
395 154 : .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT)
396 154 : .SetAvailableInPipelineStep(false);
397 154 : AddCreationOptionsArg(&m_creationOptions)
398 154 : .SetHiddenForCLI(hiddenForCLI)
399 154 : .SetAvailableInPipelineStep(false);
400 154 : AddOverwriteArg(&m_overwrite)
401 154 : .SetHiddenForCLI(hiddenForCLI)
402 154 : .SetAvailableInPipelineStep(false);
403 154 : }
404 :
405 : /************************************************************************/
406 : /* GDALPipelineStepAlgorithm::RunImpl() */
407 : /************************************************************************/
408 :
409 2795 : bool GDALPipelineStepAlgorithm::RunImpl(GDALProgressFunc pfnProgress,
410 : void *pProgressData)
411 : {
412 2795 : if (m_standaloneStep)
413 : {
414 1155 : std::unique_ptr<GDALPipelineStepAlgorithm> readAlg;
415 1155 : if (GetInputType() == GDAL_OF_RASTER)
416 732 : readAlg = std::make_unique<GDALRasterReadAlgorithm>();
417 : else
418 423 : readAlg = std::make_unique<GDALVectorReadAlgorithm>();
419 9663 : for (auto &arg : readAlg->GetArgs())
420 : {
421 8508 : auto stepArg = GetArg(arg->GetName());
422 9680 : if (stepArg && stepArg->IsExplicitlySet() &&
423 1172 : stepArg->GetType() == arg->GetType())
424 : {
425 1169 : arg->SetSkipIfAlreadySet(true);
426 1169 : arg->SetFrom(*stepArg);
427 : }
428 : }
429 :
430 0 : std::unique_ptr<GDALPipelineStepAlgorithm> writeAlg;
431 1155 : if (GetOutputType() == GDAL_OF_RASTER)
432 : {
433 516 : if (GetName() == GDALRasterInfoAlgorithm::NAME)
434 25 : writeAlg = std::make_unique<GDALRasterInfoAlgorithm>();
435 491 : else if (GetName() == GDALRasterCompareAlgorithm::NAME)
436 0 : writeAlg = std::make_unique<GDALRasterCompareAlgorithm>();
437 : else
438 491 : writeAlg = std::make_unique<GDALRasterWriteAlgorithm>();
439 : }
440 : else
441 : {
442 639 : if (GetName() == GDALVectorInfoAlgorithm::NAME)
443 31 : writeAlg = std::make_unique<GDALVectorInfoAlgorithm>();
444 : else
445 608 : writeAlg = std::make_unique<GDALVectorWriteAlgorithm>();
446 : }
447 18781 : for (auto &arg : writeAlg->GetArgs())
448 : {
449 17626 : auto stepArg = GetArg(arg->GetName());
450 20546 : if (stepArg && stepArg->IsExplicitlySet() &&
451 2920 : stepArg->GetType() == arg->GetType())
452 : {
453 2914 : arg->SetSkipIfAlreadySet(true);
454 2914 : arg->SetFrom(*stepArg);
455 : }
456 : }
457 :
458 1155 : const bool bIsStreaming = m_format == "stream";
459 :
460 : // Already checked by GDALAlgorithm::Run()
461 1155 : CPLAssert(!m_executionForStreamOutput || bIsStreaming);
462 :
463 1155 : bool ret = false;
464 1211 : if (!m_outputVRTCompatible &&
465 112 : (EQUAL(m_format.c_str(), "VRT") ||
466 56 : (m_format.empty() &&
467 1156 : EQUAL(CPLGetExtensionSafe(m_outputDataset.GetName().c_str())
468 : .c_str(),
469 : "VRT"))))
470 : {
471 0 : ReportError(CE_Failure, CPLE_NotSupported,
472 : "VRT output is not supported. Consider using the "
473 : "GDALG driver instead (files with .gdalg.json "
474 : "extension)");
475 : }
476 1155 : else if (readAlg->Run())
477 : {
478 1151 : auto outputArg = GetArg(GDAL_ARG_NAME_OUTPUT);
479 : const bool bOutputSpecified =
480 2043 : outputArg && outputArg->IsExplicitlySet() &&
481 892 : (outputArg->GetType() == GAAT_DATASET ||
482 3 : outputArg->GetType() == GAAT_DATASET_LIST);
483 :
484 1151 : m_inputDataset.clear();
485 1151 : m_inputDataset.resize(1);
486 1151 : m_inputDataset[0].Set(readAlg->m_outputDataset.GetDatasetRef());
487 1151 : if (bOutputSpecified)
488 886 : m_outputDataset.Set(nullptr);
489 :
490 : std::unique_ptr<void, decltype(&GDALDestroyScaledProgress)>
491 2302 : pScaledData(nullptr, GDALDestroyScaledProgress);
492 :
493 : const bool bCanHandleNextStep =
494 1151 : !bIsStreaming && CanHandleNextStep(writeAlg.get());
495 :
496 1151 : GDALPipelineStepRunContext stepCtxt;
497 1151 : if (pfnProgress && GetName() == GDALRasterCompareAlgorithm::NAME)
498 : {
499 5 : stepCtxt.m_pfnProgress = pfnProgress;
500 5 : stepCtxt.m_pProgressData = pProgressData;
501 : }
502 1197 : else if (pfnProgress &&
503 51 : (bCanHandleNextStep || !IsNativelyStreamingCompatible()))
504 : {
505 52 : pScaledData.reset(GDALCreateScaledProgress(
506 22 : 0.0, bIsStreaming || bCanHandleNextStep ? 1.0 : 0.5,
507 : pfnProgress, pProgressData));
508 30 : stepCtxt.m_pfnProgress =
509 30 : pScaledData ? GDALScaledProgress : nullptr;
510 30 : stepCtxt.m_pProgressData = pScaledData.get();
511 : }
512 :
513 1151 : if (bCanHandleNextStep)
514 52 : stepCtxt.m_poNextUsableStep = writeAlg.get();
515 1151 : if (RunPreStepPipelineValidations() && RunStep(stepCtxt))
516 : {
517 1066 : if (bCanHandleNextStep || !bOutputSpecified)
518 : {
519 305 : ret = true;
520 : }
521 : else
522 : {
523 761 : writeAlg->m_outputVRTCompatible = m_outputVRTCompatible;
524 761 : writeAlg->m_quiet = m_quiet;
525 :
526 1522 : std::vector<GDALArgDatasetValue> inputDataset(1);
527 761 : inputDataset[0].Set(m_outputDataset.GetDatasetRef());
528 761 : auto inputArg = writeAlg->GetArg(GDAL_ARG_NAME_INPUT);
529 761 : CPLAssert(inputArg);
530 761 : inputArg->Set(std::move(inputDataset));
531 :
532 761 : if (pfnProgress)
533 : {
534 41 : pScaledData.reset(GDALCreateScaledProgress(
535 41 : IsNativelyStreamingCompatible() ? 0.0 : 0.5, 1.0,
536 : pfnProgress, pProgressData));
537 : }
538 761 : stepCtxt.m_pfnProgress =
539 761 : pScaledData ? GDALScaledProgress : nullptr;
540 761 : stepCtxt.m_pProgressData = pScaledData.get();
541 1522 : if (writeAlg->ValidateArguments() &&
542 761 : writeAlg->RunStep(stepCtxt))
543 : {
544 742 : if (pfnProgress)
545 40 : pfnProgress(1.0, "", pProgressData);
546 :
547 742 : m_outputDataset.Set(
548 742 : writeAlg->m_outputDataset.GetDatasetRef());
549 742 : ret = true;
550 : }
551 : }
552 : }
553 : }
554 :
555 1155 : return ret;
556 : }
557 : else
558 : {
559 1640 : GDALPipelineStepRunContext stepCtxt;
560 1640 : stepCtxt.m_pfnProgress = pfnProgress;
561 1640 : stepCtxt.m_pProgressData = pProgressData;
562 1640 : return RunPreStepPipelineValidations() && RunStep(stepCtxt);
563 : }
564 : }
565 :
566 : /************************************************************************/
567 : /* SetInputDataset() */
568 : /************************************************************************/
569 :
570 27 : void GDALPipelineStepAlgorithm::SetInputDataset(GDALDataset *poDS)
571 : {
572 27 : auto arg = GetArg(GDAL_ARG_NAME_INPUT);
573 27 : if (arg)
574 : {
575 27 : auto &val = arg->Get<std::vector<GDALArgDatasetValue>>();
576 27 : val.resize(1);
577 27 : val[0].Set(poDS);
578 27 : arg->NotifyValueSet();
579 27 : arg->SetSkipIfAlreadySet();
580 : }
581 27 : }
582 :
583 : /************************************************************************/
584 : /* ProcessGDALGOutput() */
585 : /************************************************************************/
586 :
587 : GDALAlgorithm::ProcessGDALGOutputRet
588 3836 : GDALPipelineStepAlgorithm::ProcessGDALGOutput()
589 : {
590 3836 : if (m_standaloneStep)
591 : {
592 2114 : return GDALAlgorithm::ProcessGDALGOutput();
593 : }
594 : else
595 : {
596 : // GDALAbstractPipelineAlgorithm<StepAlgorithm>::RunStep() might
597 : // actually detect a GDALG output request and process it.
598 1722 : return GDALAlgorithm::ProcessGDALGOutputRet::NOT_GDALG;
599 : }
600 : }
601 :
602 : /************************************************************************/
603 : /* GDALPipelineStepAlgorithm::CheckSafeForStreamOutput() */
604 : /************************************************************************/
605 :
606 94 : bool GDALPipelineStepAlgorithm::CheckSafeForStreamOutput()
607 : {
608 94 : if (m_standaloneStep)
609 : {
610 48 : return GDALAlgorithm::CheckSafeForStreamOutput();
611 : }
612 : else
613 : {
614 : // The check is actually done in
615 : // GDALAbstractPipelineAlgorithm<StepAlgorithm>::RunStep()
616 : // so return true for now.
617 46 : return true;
618 : }
619 : }
620 :
621 : /************************************************************************/
622 : /* GDALPipelineStepAlgorithm::Finalize() */
623 : /************************************************************************/
624 :
625 1451 : bool GDALPipelineStepAlgorithm::Finalize()
626 : {
627 1451 : bool ret = GDALAlgorithm::Finalize();
628 2703 : for (auto &argValue : m_inputDataset)
629 1252 : ret = argValue.Close() && ret;
630 1451 : ret = m_outputDataset.Close() && ret;
631 1451 : return ret;
632 : }
633 :
634 : GDALAlgorithmStepRegistry::~GDALAlgorithmStepRegistry() = default;
635 :
636 : /************************************************************************/
637 : /* GDALPipelineAlgorithm */
638 : /************************************************************************/
639 :
640 331 : GDALPipelineAlgorithm::GDALPipelineAlgorithm()
641 : : GDALAbstractPipelineAlgorithm(
642 : NAME, DESCRIPTION, HELP_URL,
643 331 : ConstructorOptions().SetStandaloneStep(false))
644 : {
645 331 : m_supportsStreamedOutput = true;
646 :
647 331 : AddProgressArg();
648 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_RASTER | GDAL_OF_VECTOR,
649 331 : /* positionalAndRequired = */ false)
650 331 : .SetMinCount(1)
651 331 : .SetMaxCount(INT_MAX)
652 331 : .SetHiddenForCLI();
653 : AddOutputDatasetArg(&m_outputDataset, GDAL_OF_RASTER | GDAL_OF_VECTOR,
654 331 : /* positionalAndRequired = */ false)
655 331 : .SetHiddenForCLI()
656 331 : .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT);
657 : AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
658 331 : /* bGDALGAllowed = */ true)
659 331 : .SetHiddenForCLI();
660 662 : AddArg("pipeline", 0, _("Pipeline string or filename"), &m_pipeline)
661 331 : .SetHiddenForCLI()
662 331 : .SetPositional();
663 :
664 331 : AddOutputStringArg(&m_output).SetHiddenForCLI();
665 331 : AddStdoutArg(&m_stdout);
666 :
667 331 : AllowArbitraryLongNameArgs();
668 :
669 331 : GDALRasterPipelineAlgorithm::RegisterAlgorithms(m_stepRegistry, true);
670 331 : GDALVectorPipelineAlgorithm::RegisterAlgorithms(m_stepRegistry, true);
671 331 : m_stepRegistry.Register<GDALExternalRasterOrVectorAlgorithm>();
672 331 : m_stepRegistry.Register<GDALRasterAsFeaturesAlgorithm>();
673 331 : m_stepRegistry.Register<GDALRasterContourAlgorithm>();
674 331 : m_stepRegistry.Register<GDALRasterFootprintAlgorithm>();
675 331 : m_stepRegistry.Register<GDALRasterPixelInfoAlgorithm>();
676 331 : m_stepRegistry.Register<GDALRasterPolygonizeAlgorithm>();
677 331 : m_stepRegistry.Register<GDALRasterZonalStatsAlgorithm>();
678 331 : m_stepRegistry.Register<GDALVectorGridAlgorithm>();
679 331 : m_stepRegistry.Register<GDALVectorRasterizeAlgorithm>();
680 331 : }
681 :
682 : /************************************************************************/
683 : /* GDALPipelineAlgorithm::GetUsageForCLI() */
684 : /************************************************************************/
685 :
686 : std::string
687 19 : GDALPipelineAlgorithm::GetUsageForCLI(bool shortUsage,
688 : const UsageOptions &usageOptions) const
689 : {
690 19 : UsageOptions stepUsageOptions;
691 19 : stepUsageOptions.isPipelineStep = true;
692 :
693 19 : if (!m_helpDocCategory.empty() && m_helpDocCategory != "main")
694 : {
695 4 : auto alg = GetStepAlg(m_helpDocCategory);
696 2 : if (alg)
697 : {
698 3 : alg->SetCallPath({CPLString(m_helpDocCategory)
699 2 : .replaceAll(RASTER_SUFFIX, "")
700 3 : .replaceAll(VECTOR_SUFFIX, "")});
701 1 : alg->GetArg("help-doc")->Set(true);
702 1 : return alg->GetUsageForCLI(shortUsage, stepUsageOptions);
703 : }
704 : else
705 : {
706 1 : fprintf(stderr, "ERROR: unknown pipeline step '%s'\n",
707 : m_helpDocCategory.c_str());
708 : return CPLSPrintf("ERROR: unknown pipeline step '%s'\n",
709 1 : m_helpDocCategory.c_str());
710 : }
711 : }
712 :
713 17 : UsageOptions usageOptionsMain(usageOptions);
714 17 : usageOptionsMain.isPipelineMain = true;
715 : std::string ret =
716 34 : GDALAlgorithm::GetUsageForCLI(shortUsage, usageOptionsMain);
717 17 : if (shortUsage)
718 15 : return ret;
719 :
720 : ret +=
721 : "\n<PIPELINE> is of the form: read|calc|concat|create|mosaic|stack "
722 : "[READ-OPTIONS] "
723 2 : "( ! <STEP-NAME> [STEP-OPTIONS] )* ! write!info!tile [WRITE-OPTIONS]\n";
724 :
725 2 : if (m_helpDocCategory == "main")
726 : {
727 1 : return ret;
728 : }
729 :
730 1 : ret += '\n';
731 1 : ret += "Example: 'gdal pipeline --progress ! read in.tif ! \\\n";
732 1 : ret += " rasterize --size 256 256 ! buffer 20 ! ";
733 1 : ret += "write out.gpkg --overwrite'\n";
734 1 : ret += '\n';
735 1 : ret += "Potential steps are:\n";
736 :
737 84 : for (const std::string &name : m_stepRegistry.GetNames())
738 : {
739 166 : auto alg = GetStepAlg(name);
740 83 : assert(alg);
741 83 : auto [options, maxOptLen] = alg->GetArgNamesForCLI();
742 83 : stepUsageOptions.maxOptLen =
743 83 : std::max(stepUsageOptions.maxOptLen, maxOptLen);
744 : }
745 :
746 : {
747 1 : ret += '\n';
748 1 : auto alg = std::make_unique<GDALRasterReadAlgorithm>();
749 2 : alg->SetCallPath({alg->GetName()});
750 1 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
751 : }
752 : {
753 1 : ret += '\n';
754 1 : auto alg = std::make_unique<GDALVectorReadAlgorithm>();
755 2 : alg->SetCallPath({alg->GetName()});
756 1 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
757 : }
758 84 : for (const std::string &name : m_stepRegistry.GetNames())
759 : {
760 166 : auto alg = GetStepAlg(name);
761 83 : assert(alg);
762 92 : if (alg->CanBeFirstStep() && !alg->CanBeMiddleStep() &&
763 98 : !alg->IsHidden() &&
764 6 : !STARTS_WITH(name.c_str(), GDALRasterReadAlgorithm::NAME))
765 : {
766 4 : ret += '\n';
767 8 : alg->SetCallPath({name});
768 4 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
769 : }
770 : }
771 84 : for (const std::string &name : m_stepRegistry.GetNames())
772 : {
773 166 : auto alg = GetStepAlg(name);
774 83 : assert(alg);
775 83 : if (alg->CanBeMiddleStep() && !alg->IsHidden())
776 : {
777 69 : ret += '\n';
778 207 : alg->SetCallPath({CPLString(alg->GetName())
779 138 : .replaceAll(RASTER_SUFFIX, "")
780 207 : .replaceAll(VECTOR_SUFFIX, "")});
781 69 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
782 : }
783 : }
784 84 : for (const std::string &name : m_stepRegistry.GetNames())
785 : {
786 166 : auto alg = GetStepAlg(name);
787 83 : assert(alg);
788 96 : if (alg->CanBeLastStep() && !alg->CanBeMiddleStep() &&
789 104 : !alg->IsHidden() &&
790 8 : !STARTS_WITH(name.c_str(), GDALRasterWriteAlgorithm::NAME))
791 : {
792 6 : ret += '\n';
793 12 : alg->SetCallPath({name});
794 6 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
795 : }
796 : }
797 : {
798 1 : ret += '\n';
799 1 : auto alg = std::make_unique<GDALRasterWriteAlgorithm>();
800 2 : alg->SetCallPath({alg->GetName()});
801 1 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
802 : }
803 : {
804 1 : ret += '\n';
805 1 : auto alg = std::make_unique<GDALVectorWriteAlgorithm>();
806 2 : alg->SetCallPath({alg->GetName()});
807 1 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
808 : }
809 :
810 1 : ret += GetUsageForCLIEnd();
811 :
812 1 : return ret;
813 : }
814 :
815 : //! @endcond
|
