Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: GDAL
4 : * Purpose: gdal "pipeline" subcommand
5 : * Author: Even Rouault <even dot rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2024, Even Rouault <even dot rouault at spatialys.com>
9 : *
10 : * SPDX-License-Identifier: MIT
11 : ****************************************************************************/
12 :
13 : //! @cond Doxygen_Suppress
14 :
15 : #include "gdalalg_pipeline.h"
16 : #include "cpl_error.h"
17 : #include "gdal_priv.h"
18 :
19 : #include "gdalalg_external.h"
20 :
21 : #include "gdalalg_raster_read.h"
22 : #include "gdalalg_raster_mosaic.h"
23 : #include "gdalalg_raster_stack.h"
24 : #include "gdalalg_raster_write.h"
25 : #include "gdalalg_raster_zonal_stats.h"
26 :
27 : #include "gdalalg_vector_read.h"
28 : #include "gdalalg_vector_write.h"
29 :
30 : #include "gdalalg_raster_as_features.h"
31 : #include "gdalalg_raster_compare.h"
32 : #include "gdalalg_raster_contour.h"
33 : #include "gdalalg_raster_footprint.h"
34 : #include "gdalalg_raster_polygonize.h"
35 : #include "gdalalg_raster_info.h"
36 : #include "gdalalg_raster_pixel_info.h"
37 : #include "gdalalg_raster_tile.h"
38 : #include "gdalalg_vector_grid.h"
39 : #include "gdalalg_vector_info.h"
40 : #include "gdalalg_vector_rasterize.h"
41 :
42 : #include <algorithm>
43 : #include <cassert>
44 :
45 : #ifndef _
46 : #define _(x) (x)
47 : #endif
48 :
49 : /************************************************************************/
50 : /* GDALPipelineStepAlgorithm() */
51 : /************************************************************************/
52 :
53 14437 : GDALPipelineStepAlgorithm::GDALPipelineStepAlgorithm(
54 : const std::string &name, const std::string &description,
55 14437 : const std::string &helpURL, const ConstructorOptions &options)
56 : : GDALAlgorithm(name, description, helpURL),
57 14437 : m_standaloneStep(options.standaloneStep), m_constructorOptions(options)
58 : {
59 14437 : }
60 :
61 : /************************************************************************/
62 : /* GDALPipelineStepAlgorithm::AddRasterHiddenInputDatasetArg() */
63 : /************************************************************************/
64 :
65 2191 : void GDALPipelineStepAlgorithm::AddRasterHiddenInputDatasetArg()
66 : {
67 2191 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_RASTER, false)
68 2191 : .SetMinCount(0)
69 2191 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
70 2191 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
71 2191 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
72 2191 : .SetHidden();
73 2191 : }
74 :
75 : /************************************************************************/
76 : /* GDALPipelineStepAlgorithm::AddRasterInputArgs() */
77 : /************************************************************************/
78 :
79 4786 : void GDALPipelineStepAlgorithm::AddRasterInputArgs(
80 : bool openForMixedRasterVector, bool hiddenForCLI)
81 : {
82 4786 : AddInputFormatsArg(&m_inputFormats)
83 : .AddMetadataItem(
84 : GAAMDI_REQUIRED_CAPABILITIES,
85 : openForMixedRasterVector
86 14699 : ? std::vector<std::string>{GDAL_DCAP_RASTER, GDAL_DCAP_VECTOR}
87 14017 : : std::vector<std::string>{GDAL_DCAP_RASTER})
88 4786 : .SetHiddenForCLI(hiddenForCLI)
89 4786 : .SetAvailableInPipelineStep(false);
90 4786 : AddOpenOptionsArg(&m_openOptions)
91 4786 : .SetHiddenForCLI(hiddenForCLI)
92 4786 : .SetAvailableInPipelineStep(false);
93 :
94 4786 : const int nDatasetType = openForMixedRasterVector
95 4786 : ? (GDAL_OF_RASTER | GDAL_OF_VECTOR)
96 : : GDAL_OF_RASTER;
97 : auto &arg =
98 : AddInputDatasetArg(
99 : &m_inputDataset, nDatasetType, false,
100 9148 : m_constructorOptions.inputDatasetHelpMsg.empty() &&
101 4362 : m_constructorOptions.inputDatasetMaxCount == 1
102 3999 : ? CPLSPrintf(
103 : "Input %s dataset",
104 8785 : GDALAlgorithmArgDatasetTypeName(nDatasetType).c_str())
105 13147 : : m_constructorOptions.inputDatasetHelpMsg.c_str())
106 4786 : .SetDatasetInputFlags(m_constructorOptions.inputDatasetInputFlags)
107 4786 : .SetMinCount(m_constructorOptions.inputDatasetRequired ? 1 : 0)
108 4786 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
109 4786 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
110 4786 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
111 4786 : .SetHiddenForCLI(hiddenForCLI)
112 4786 : .SetAvailableInPipelineStep(false);
113 4786 : if (m_constructorOptions.inputDatasetPositional && !hiddenForCLI)
114 4454 : arg.SetPositional();
115 4786 : if (m_constructorOptions.inputDatasetRequired && !hiddenForCLI)
116 4454 : arg.SetRequired();
117 4786 : if (!m_constructorOptions.inputDatasetAlias.empty())
118 841 : arg.AddAlias(m_constructorOptions.inputDatasetAlias);
119 4786 : }
120 :
121 : /************************************************************************/
122 : /* GDALPipelineStepAlgorithm::AddRasterOutputArgs() */
123 : /************************************************************************/
124 :
125 3063 : void GDALPipelineStepAlgorithm::AddRasterOutputArgs(bool hiddenForCLI)
126 : {
127 3063 : m_outputFormatArg =
128 : &(AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
129 3063 : /* bGDALGAllowed = */ true)
130 : .AddMetadataItem(
131 : GAAMDI_REQUIRED_CAPABILITIES,
132 : {GDAL_DCAP_RASTER,
133 12252 : m_constructorOptions.outputFormatCreateCapability.c_str()})
134 3063 : .SetHiddenForCLI(hiddenForCLI))
135 3063 : .SetAvailableInPipelineStep(false);
136 : AddOutputDatasetArg(&m_outputDataset, GDAL_OF_RASTER,
137 3063 : /* positionalAndRequired = */ !hiddenForCLI,
138 3063 : m_constructorOptions.outputDatasetHelpMsg.c_str())
139 3063 : .SetHiddenForCLI(hiddenForCLI)
140 3063 : .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT)
141 3063 : .SetAvailableInPipelineStep(false);
142 3063 : AddCreationOptionsArg(&m_creationOptions)
143 3063 : .SetHiddenForCLI(hiddenForCLI)
144 3063 : .SetAvailableInPipelineStep(false);
145 3063 : constexpr const char *MUTUAL_EXCLUSION_GROUP_OVERWRITE_APPEND =
146 : "overwrite-append";
147 3063 : AddOverwriteArg(&m_overwrite)
148 3063 : .SetHiddenForCLI(hiddenForCLI)
149 6126 : .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_OVERWRITE_APPEND)
150 3063 : .SetAvailableInPipelineStep(false);
151 : AddArg(GDAL_ARG_NAME_APPEND, 0,
152 6126 : _("Append as a subdataset to existing output"), &m_appendRaster)
153 3063 : .SetDefault(false)
154 3063 : .SetHiddenForCLI(hiddenForCLI)
155 6126 : .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_OVERWRITE_APPEND)
156 3063 : .SetAvailableInPipelineStep(false);
157 3063 : }
158 :
159 : /************************************************************************/
160 : /* GDALPipelineStepAlgorithm::AddVectorHiddenInputDatasetArg() */
161 : /************************************************************************/
162 :
163 2116 : void GDALPipelineStepAlgorithm::AddVectorHiddenInputDatasetArg()
164 : {
165 2116 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_VECTOR, false)
166 2116 : .SetMinCount(0)
167 2116 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
168 2116 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
169 2116 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
170 2116 : .SetHidden();
171 2116 : }
172 :
173 : /************************************************************************/
174 : /* GDALPipelineStepAlgorithm::AddVectorInputArgs() */
175 : /************************************************************************/
176 :
177 2917 : void GDALPipelineStepAlgorithm::AddVectorInputArgs(bool hiddenForCLI)
178 : {
179 2917 : AddInputFormatsArg(&m_inputFormats)
180 8751 : .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES, {GDAL_DCAP_VECTOR})
181 2917 : .SetHiddenForCLI(hiddenForCLI)
182 2917 : .SetAvailableInPipelineStep(false);
183 2917 : AddOpenOptionsArg(&m_openOptions)
184 2917 : .SetHiddenForCLI(hiddenForCLI)
185 2917 : .SetAvailableInPipelineStep(false);
186 : auto &datasetArg =
187 : AddInputDatasetArg(
188 : &m_inputDataset, GDAL_OF_VECTOR, false,
189 8620 : m_constructorOptions.inputDatasetHelpMsg.empty() &&
190 2786 : m_constructorOptions.inputDatasetMaxCount == 1
191 : ? "Input vector dataset"
192 5703 : : m_constructorOptions.inputDatasetHelpMsg.c_str())
193 2917 : .SetMinCount(m_constructorOptions.inputDatasetRequired ? 1 : 0)
194 2917 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
195 2917 : .SetDatasetInputFlags(m_constructorOptions.inputDatasetInputFlags)
196 2917 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
197 2917 : .SetHiddenForCLI(hiddenForCLI)
198 2917 : .SetAvailableInPipelineStep(false);
199 2917 : if (!m_constructorOptions.inputDatasetAlias.empty())
200 190 : datasetArg.AddAlias(m_constructorOptions.inputDatasetAlias);
201 2917 : if (!m_constructorOptions.inputDatasetMetaVar.empty())
202 2917 : datasetArg.SetMetaVar(m_constructorOptions.inputDatasetMetaVar);
203 2917 : if (m_constructorOptions.inputDatasetPositional && !hiddenForCLI)
204 2616 : datasetArg.SetPositional();
205 2917 : if (m_constructorOptions.inputDatasetRequired && !hiddenForCLI)
206 2616 : datasetArg.SetRequired();
207 2917 : if (m_constructorOptions.addInputLayerNameArgument)
208 : {
209 : auto &layerArg = AddArg(GDAL_ARG_NAME_INPUT_LAYER, 'l',
210 5138 : _("Input layer name(s)"), &m_inputLayerNames)
211 5138 : .AddAlias("layer")
212 2569 : .SetHiddenForCLI(hiddenForCLI)
213 2569 : .SetAvailableInPipelineStep(false);
214 2569 : SetAutoCompleteFunctionForLayerName(layerArg, datasetArg);
215 : }
216 2917 : }
217 :
218 : /************************************************************************/
219 : /* GDALPipelineStepAlgorithm::AddVectorOutputArgs() */
220 : /************************************************************************/
221 :
222 3330 : void GDALPipelineStepAlgorithm::AddVectorOutputArgs(
223 : bool hiddenForCLI, bool shortNameOutputLayerAllowed)
224 : {
225 : AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
226 3330 : /* bGDALGAllowed = */ true)
227 : .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,
228 13320 : {GDAL_DCAP_VECTOR, GDAL_DCAP_CREATE})
229 3330 : .SetHiddenForCLI(hiddenForCLI)
230 3330 : .SetAvailableInPipelineStep(false);
231 3330 : AddOutputOpenOptionsArg(&m_outputOpenOptions)
232 3330 : .SetHiddenForCLI(hiddenForCLI)
233 3330 : .SetAvailableInPipelineStep(false);
234 : auto &outputDatasetArg =
235 : AddOutputDatasetArg(&m_outputDataset, GDAL_OF_VECTOR,
236 3330 : /* positionalAndRequired = */ false)
237 3330 : .SetHiddenForCLI(hiddenForCLI)
238 3330 : .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT)
239 3330 : .SetAvailableInPipelineStep(false);
240 3330 : if (!hiddenForCLI)
241 3160 : outputDatasetArg.SetPositional();
242 3330 : if (!hiddenForCLI && m_constructorOptions.outputDatasetRequired)
243 3101 : outputDatasetArg.SetRequired();
244 :
245 3330 : AddCreationOptionsArg(&m_creationOptions)
246 3330 : .SetHiddenForCLI(hiddenForCLI)
247 3330 : .SetAvailableInPipelineStep(false);
248 3330 : AddLayerCreationOptionsArg(&m_layerCreationOptions)
249 3330 : .SetHiddenForCLI(hiddenForCLI)
250 3330 : .SetAvailableInPipelineStep(false);
251 3330 : AddOverwriteArg(&m_overwrite)
252 3330 : .SetHiddenForCLI(hiddenForCLI)
253 3330 : .SetAvailableInPipelineStep(false);
254 3330 : GDALInConstructionAlgorithmArg *updateArg = nullptr;
255 3330 : if (m_constructorOptions.addUpdateArgument)
256 : {
257 3268 : updateArg = &AddUpdateArg(&m_update)
258 3268 : .SetHiddenForCLI(hiddenForCLI)
259 3268 : .SetAvailableInPipelineStep(false);
260 : }
261 3330 : if (m_constructorOptions.addOverwriteLayerArgument)
262 : {
263 3268 : AddOverwriteLayerArg(&m_overwriteLayer)
264 3268 : .SetHiddenForCLI(hiddenForCLI)
265 3268 : .SetAvailableInPipelineStep(false);
266 : }
267 3330 : constexpr const char *MUTUAL_EXCLUSION_GROUP_APPEND_UPSERT =
268 : "append-upsert";
269 3330 : if (m_constructorOptions.addAppendLayerArgument)
270 : {
271 3137 : AddAppendLayerArg(&m_appendLayer)
272 3137 : .SetHiddenForCLI(hiddenForCLI)
273 6274 : .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_APPEND_UPSERT)
274 3137 : .SetAvailableInPipelineStep(false);
275 : }
276 3330 : if (m_constructorOptions.addUpsertArgument)
277 : {
278 5860 : AddArg("upsert", 0, _("Upsert features (implies 'append')"), &m_upsert)
279 2930 : .SetHiddenForCLI(hiddenForCLI)
280 5860 : .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_APPEND_UPSERT)
281 :
282 2930 : .SetAvailableInPipelineStep(false)
283 : .AddAction(
284 12 : [updateArg, this]()
285 : {
286 4 : if (m_upsert && updateArg)
287 4 : updateArg->Set(true);
288 5860 : })
289 2930 : .SetCategory(GAAC_ADVANCED);
290 : }
291 3330 : if (m_constructorOptions.addOutputLayerNameArgument)
292 : {
293 3202 : AddOutputLayerNameArg(hiddenForCLI, shortNameOutputLayerAllowed);
294 : }
295 3330 : if (m_constructorOptions.addSkipErrorsArgument)
296 : {
297 : AddArg("skip-errors", 0, _("Skip errors when writing features"),
298 5860 : &m_skipErrors)
299 5860 : .AddHiddenAlias("skip-failures") // For ogr2ogr nostalgic people
300 2930 : .SetAvailableInPipelineStep(false);
301 : }
302 3330 : if (m_constructorOptions.addNoCreateEmptyLayersArgument)
303 : {
304 : AddArg("no-create-empty-layers", 0,
305 : _("Avoid creating layers to which no features will be written"),
306 2210 : &m_noCreateEmptyLayers)
307 1105 : .SetAvailableInPipelineStep(false);
308 : }
309 3330 : }
310 :
311 : /************************************************************************/
312 : /* GDALPipelineStepAlgorithm::AddOutputLayerNameArg() */
313 : /************************************************************************/
314 :
315 3443 : void GDALPipelineStepAlgorithm::AddOutputLayerNameArg(
316 : bool hiddenForCLI, bool shortNameOutputLayerAllowed)
317 : {
318 : AddArg(GDAL_ARG_NAME_OUTPUT_LAYER, shortNameOutputLayerAllowed ? 'l' : 0,
319 : _("Output layer name"),
320 6886 : &m_outputLayerName)
321 6886 : .AddHiddenAlias("nln") // For ogr2ogr nostalgic people
322 3443 : .SetHiddenForCLI(hiddenForCLI)
323 : .SetAvailableInPipelineStep(
324 3443 : m_constructorOptions.outputLayerNameAvailableInPipelineStep);
325 3443 : }
326 :
327 : /************************************************************************/
328 : /* GDALPipelineStepAlgorithm::AddMdimHiddenInputDatasetArg() */
329 : /************************************************************************/
330 :
331 8 : void GDALPipelineStepAlgorithm::AddMdimHiddenInputDatasetArg()
332 : {
333 8 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_MULTIDIM_RASTER, false)
334 8 : .SetMinCount(0)
335 8 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
336 8 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
337 8 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
338 8 : .SetHidden();
339 8 : }
340 :
341 : /************************************************************************/
342 : /* GDALPipelineStepAlgorithm::AddMdimInputArgs() */
343 : /************************************************************************/
344 :
345 205 : void GDALPipelineStepAlgorithm::AddMdimInputArgs(bool openForMixedMdimVector,
346 : bool hiddenForCLI,
347 : bool acceptRaster)
348 : {
349 205 : AddInputFormatsArg(&m_inputFormats)
350 : .AddMetadataItem(
351 : GAAMDI_REQUIRED_CAPABILITIES,
352 : openForMixedMdimVector
353 615 : ? std::vector<
354 : std::string>{acceptRaster
355 : ? GDAL_ALG_DCAP_RASTER_OR_MULTIDIM_RASTER
356 : : GDAL_DCAP_MULTIDIM_RASTER,
357 0 : GDAL_DCAP_VECTOR}
358 : : std::vector<
359 : std::string>{acceptRaster
360 : ? GDAL_ALG_DCAP_RASTER_OR_MULTIDIM_RASTER
361 615 : : GDAL_DCAP_MULTIDIM_RASTER})
362 205 : .SetHiddenForCLI(hiddenForCLI)
363 205 : .SetAvailableInPipelineStep(false);
364 205 : AddOpenOptionsArg(&m_openOptions)
365 205 : .SetHiddenForCLI(hiddenForCLI)
366 205 : .SetAvailableInPipelineStep(false);
367 : auto &arg =
368 : AddInputDatasetArg(&m_inputDataset,
369 205 : (acceptRaster ? GDAL_OF_RASTER : 0) |
370 : (openForMixedMdimVector
371 205 : ? (GDAL_OF_MULTIDIM_RASTER | GDAL_OF_VECTOR)
372 : : GDAL_OF_MULTIDIM_RASTER),
373 : false,
374 410 : m_constructorOptions.inputDatasetHelpMsg.c_str())
375 205 : .SetDatasetInputFlags(m_constructorOptions.inputDatasetInputFlags)
376 205 : .SetMinCount(m_constructorOptions.inputDatasetRequired ? 1 : 0)
377 205 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
378 205 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
379 205 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
380 205 : .SetHiddenForCLI(hiddenForCLI)
381 205 : .SetAvailableInPipelineStep(false);
382 205 : if (m_constructorOptions.inputDatasetPositional && !hiddenForCLI)
383 156 : arg.SetPositional();
384 205 : if (m_constructorOptions.inputDatasetRequired && !hiddenForCLI)
385 156 : arg.SetRequired();
386 205 : if (!m_constructorOptions.inputDatasetAlias.empty())
387 50 : arg.AddAlias(m_constructorOptions.inputDatasetAlias);
388 205 : }
389 :
390 : /************************************************************************/
391 : /* GDALPipelineStepAlgorithm::AddMdimOutputArgs() */
392 : /************************************************************************/
393 :
394 154 : void GDALPipelineStepAlgorithm::AddMdimOutputArgs(bool hiddenForCLI)
395 : {
396 154 : m_outputFormatArg =
397 : &(AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
398 154 : /* bGDALGAllowed = */ true)
399 : .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,
400 462 : {GDAL_DCAP_CREATE_MULTIDIMENSIONAL})
401 154 : .SetHiddenForCLI(hiddenForCLI))
402 154 : .SetAvailableInPipelineStep(false);
403 : AddOutputDatasetArg(&m_outputDataset, GDAL_OF_MULTIDIM_RASTER,
404 154 : /* positionalAndRequired = */ !hiddenForCLI,
405 154 : m_constructorOptions.outputDatasetHelpMsg.c_str())
406 154 : .SetHiddenForCLI(hiddenForCLI)
407 154 : .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT)
408 154 : .SetAvailableInPipelineStep(false);
409 154 : AddCreationOptionsArg(&m_creationOptions)
410 154 : .SetHiddenForCLI(hiddenForCLI)
411 154 : .SetAvailableInPipelineStep(false);
412 154 : AddOverwriteArg(&m_overwrite)
413 154 : .SetHiddenForCLI(hiddenForCLI)
414 154 : .SetAvailableInPipelineStep(false);
415 154 : }
416 :
417 : /************************************************************************/
418 : /* GDALPipelineStepAlgorithm::RunImpl() */
419 : /************************************************************************/
420 :
421 2845 : bool GDALPipelineStepAlgorithm::RunImpl(GDALProgressFunc pfnProgress,
422 : void *pProgressData)
423 : {
424 2845 : if (m_standaloneStep)
425 : {
426 1177 : std::unique_ptr<GDALPipelineStepAlgorithm> readAlg;
427 1177 : if (GetInputType() == GDAL_OF_RASTER)
428 732 : readAlg = std::make_unique<GDALRasterReadAlgorithm>();
429 : else
430 445 : readAlg = std::make_unique<GDALVectorReadAlgorithm>();
431 9861 : for (auto &arg : readAlg->GetArgs())
432 : {
433 8684 : auto stepArg = GetArg(arg->GetName());
434 9878 : if (stepArg && stepArg->IsExplicitlySet() &&
435 1194 : stepArg->GetType() == arg->GetType())
436 : {
437 1191 : arg->SetSkipIfAlreadySet(true);
438 1191 : arg->SetFrom(*stepArg);
439 : }
440 : }
441 :
442 0 : std::unique_ptr<GDALPipelineStepAlgorithm> writeAlg;
443 1177 : if (GetOutputType() == GDAL_OF_RASTER)
444 : {
445 516 : if (GetName() == GDALRasterInfoAlgorithm::NAME)
446 25 : writeAlg = std::make_unique<GDALRasterInfoAlgorithm>();
447 491 : else if (GetName() == GDALRasterCompareAlgorithm::NAME)
448 0 : writeAlg = std::make_unique<GDALRasterCompareAlgorithm>();
449 : else
450 491 : writeAlg = std::make_unique<GDALRasterWriteAlgorithm>();
451 : }
452 : else
453 : {
454 661 : if (GetName() == GDALVectorInfoAlgorithm::NAME)
455 31 : writeAlg = std::make_unique<GDALVectorInfoAlgorithm>();
456 : else
457 630 : writeAlg = std::make_unique<GDALVectorWriteAlgorithm>();
458 : }
459 19221 : for (auto &arg : writeAlg->GetArgs())
460 : {
461 18044 : auto stepArg = GetArg(arg->GetName());
462 21028 : if (stepArg && stepArg->IsExplicitlySet() &&
463 2984 : stepArg->GetType() == arg->GetType())
464 : {
465 2978 : arg->SetSkipIfAlreadySet(true);
466 2978 : arg->SetFrom(*stepArg);
467 : }
468 : }
469 :
470 1177 : const bool bIsStreaming = m_format == "stream";
471 :
472 : // Already checked by GDALAlgorithm::Run()
473 1177 : CPLAssert(!m_executionForStreamOutput || bIsStreaming);
474 :
475 1177 : bool ret = false;
476 1233 : if (!m_outputVRTCompatible &&
477 112 : (EQUAL(m_format.c_str(), "VRT") ||
478 56 : (m_format.empty() &&
479 1178 : EQUAL(CPLGetExtensionSafe(m_outputDataset.GetName().c_str())
480 : .c_str(),
481 : "VRT"))))
482 : {
483 0 : ReportError(CE_Failure, CPLE_NotSupported,
484 : "VRT output is not supported. Consider using the "
485 : "GDALG driver instead (files with .gdalg.json "
486 : "extension)");
487 : }
488 1177 : else if (readAlg->Run())
489 : {
490 1173 : auto outputArg = GetArg(GDAL_ARG_NAME_OUTPUT);
491 : const bool bOutputSpecified =
492 2087 : outputArg && outputArg->IsExplicitlySet() &&
493 914 : (outputArg->GetType() == GAAT_DATASET ||
494 3 : outputArg->GetType() == GAAT_DATASET_LIST);
495 :
496 1173 : m_inputDataset.clear();
497 1173 : m_inputDataset.resize(1);
498 1173 : m_inputDataset[0].Set(readAlg->m_outputDataset.GetDatasetRef());
499 1173 : if (bOutputSpecified)
500 908 : m_outputDataset.Set(nullptr);
501 :
502 : std::unique_ptr<void, decltype(&GDALDestroyScaledProgress)>
503 2346 : pScaledData(nullptr, GDALDestroyScaledProgress);
504 :
505 : const bool bCanHandleNextStep =
506 1173 : !bIsStreaming && CanHandleNextStep(writeAlg.get());
507 :
508 1173 : GDALPipelineStepRunContext stepCtxt;
509 1173 : if (pfnProgress && GetName() == GDALRasterCompareAlgorithm::NAME)
510 : {
511 5 : stepCtxt.m_pfnProgress = pfnProgress;
512 5 : stepCtxt.m_pProgressData = pProgressData;
513 : }
514 1219 : else if (pfnProgress &&
515 51 : (bCanHandleNextStep || !IsNativelyStreamingCompatible()))
516 : {
517 52 : pScaledData.reset(GDALCreateScaledProgress(
518 22 : 0.0, bIsStreaming || bCanHandleNextStep ? 1.0 : 0.5,
519 : pfnProgress, pProgressData));
520 30 : stepCtxt.m_pfnProgress =
521 30 : pScaledData ? GDALScaledProgress : nullptr;
522 30 : stepCtxt.m_pProgressData = pScaledData.get();
523 : }
524 :
525 1173 : if (bCanHandleNextStep)
526 52 : stepCtxt.m_poNextUsableStep = writeAlg.get();
527 1173 : if (RunPreStepPipelineValidations() && RunStep(stepCtxt))
528 : {
529 1087 : if (bCanHandleNextStep || !bOutputSpecified)
530 : {
531 305 : ret = true;
532 : }
533 : else
534 : {
535 782 : writeAlg->m_outputVRTCompatible = m_outputVRTCompatible;
536 782 : writeAlg->m_quiet = m_quiet;
537 :
538 1564 : std::vector<GDALArgDatasetValue> inputDataset(1);
539 782 : inputDataset[0].Set(m_outputDataset.GetDatasetRef());
540 782 : auto inputArg = writeAlg->GetArg(GDAL_ARG_NAME_INPUT);
541 782 : CPLAssert(inputArg);
542 782 : inputArg->Set(std::move(inputDataset));
543 :
544 782 : if (pfnProgress)
545 : {
546 41 : pScaledData.reset(GDALCreateScaledProgress(
547 41 : IsNativelyStreamingCompatible() ? 0.0 : 0.5, 1.0,
548 : pfnProgress, pProgressData));
549 : }
550 782 : stepCtxt.m_pfnProgress =
551 782 : pScaledData ? GDALScaledProgress : nullptr;
552 782 : stepCtxt.m_pProgressData = pScaledData.get();
553 1564 : if (writeAlg->ValidateArguments() &&
554 782 : writeAlg->RunStep(stepCtxt))
555 : {
556 756 : if (pfnProgress)
557 40 : pfnProgress(1.0, "", pProgressData);
558 :
559 756 : m_outputDataset.Set(
560 756 : writeAlg->m_outputDataset.GetDatasetRef());
561 756 : ret = true;
562 : }
563 : }
564 : }
565 : }
566 :
567 1177 : return ret;
568 : }
569 : else
570 : {
571 1668 : GDALPipelineStepRunContext stepCtxt;
572 1668 : stepCtxt.m_pfnProgress = pfnProgress;
573 1668 : stepCtxt.m_pProgressData = pProgressData;
574 1668 : return RunPreStepPipelineValidations() && RunStep(stepCtxt);
575 : }
576 : }
577 :
578 : /************************************************************************/
579 : /* SetInputDataset() */
580 : /************************************************************************/
581 :
582 27 : void GDALPipelineStepAlgorithm::SetInputDataset(GDALDataset *poDS)
583 : {
584 27 : auto arg = GetArg(GDAL_ARG_NAME_INPUT);
585 27 : if (arg)
586 : {
587 27 : auto &val = arg->Get<std::vector<GDALArgDatasetValue>>();
588 27 : val.resize(1);
589 27 : val[0].Set(poDS);
590 27 : arg->NotifyValueSet();
591 27 : arg->SetSkipIfAlreadySet();
592 : }
593 27 : }
594 :
595 : /************************************************************************/
596 : /* ProcessGDALGOutput() */
597 : /************************************************************************/
598 :
599 : GDALAlgorithm::ProcessGDALGOutputRet
600 3939 : GDALPipelineStepAlgorithm::ProcessGDALGOutput()
601 : {
602 3939 : if (m_standaloneStep)
603 : {
604 2189 : return GDALAlgorithm::ProcessGDALGOutput();
605 : }
606 : else
607 : {
608 : // GDALAbstractPipelineAlgorithm<StepAlgorithm>::RunStep() might
609 : // actually detect a GDALG output request and process it.
610 1750 : return GDALAlgorithm::ProcessGDALGOutputRet::NOT_GDALG;
611 : }
612 : }
613 :
614 : /************************************************************************/
615 : /* GDALPipelineStepAlgorithm::CheckSafeForStreamOutput() */
616 : /************************************************************************/
617 :
618 96 : bool GDALPipelineStepAlgorithm::CheckSafeForStreamOutput()
619 : {
620 96 : if (m_standaloneStep)
621 : {
622 50 : return GDALAlgorithm::CheckSafeForStreamOutput();
623 : }
624 : else
625 : {
626 : // The check is actually done in
627 : // GDALAbstractPipelineAlgorithm<StepAlgorithm>::RunStep()
628 : // so return true for now.
629 46 : return true;
630 : }
631 : }
632 :
633 : /************************************************************************/
634 : /* GDALPipelineStepAlgorithm::Finalize() */
635 : /************************************************************************/
636 :
637 1475 : bool GDALPipelineStepAlgorithm::Finalize()
638 : {
639 1475 : bool ret = GDALAlgorithm::Finalize();
640 2748 : for (auto &argValue : m_inputDataset)
641 1273 : ret = argValue.Close() && ret;
642 1475 : ret = m_outputDataset.Close() && ret;
643 1475 : return ret;
644 : }
645 :
646 : /************************************************************************/
647 : /* CreateDatasetSingleOutputLayerIfNeeded() */
648 : /************************************************************************/
649 :
650 64 : bool GDALPipelineStepAlgorithm::CreateDatasetSingleOutputLayerIfNeeded(
651 : GDALPipelineStepRunContext &ctxt, const std::string &defaultLayerName,
652 : GDALDataset *&poDstDS, bool &bTemporaryFile,
653 : std::unique_ptr<GDALDataset> &poNewRetDS, std::string &outputLayerName,
654 : OGRLayer *&poDstLayer)
655 : {
656 64 : auto poWriteStep = ctxt.m_poNextUsableStep ? ctxt.m_poNextUsableStep : this;
657 128 : std::string outputFilename = poWriteStep->GetOutputDataset().GetName();
658 :
659 64 : poDstDS = poWriteStep->GetOutputDataset().GetDatasetRef();
660 64 : poNewRetDS.reset();
661 64 : bTemporaryFile = false;
662 64 : poDstLayer = nullptr;
663 64 : GDALDriver *poDstDriver = nullptr;
664 64 : if (!poDstDS)
665 : {
666 44 : auto poDriverManager = GetGDALDriverManager();
667 44 : std::string format = poWriteStep->GetOutputFormat();
668 44 : if (m_standaloneStep || (ctxt.m_poNextUsableStep && format.empty()))
669 : {
670 37 : if (format.empty())
671 : {
672 : const auto aosFormats =
673 : CPLStringList(GDALGetOutputDriversForDatasetName(
674 : outputFilename.c_str(), GDAL_OF_VECTOR,
675 : /* bSingleMatch = */ true,
676 25 : /* bWarn = */ true));
677 25 : if (aosFormats.size() != 1)
678 : {
679 2 : ReportError(CE_Failure, CPLE_AppDefined,
680 : "Cannot guess driver for %s",
681 : outputFilename.c_str());
682 2 : return false;
683 : }
684 23 : format = aosFormats[0];
685 : }
686 : }
687 7 : else if (!ctxt.m_poNextUsableStep)
688 : {
689 5 : poDstDriver = poDriverManager->GetDriverByName("GPKG");
690 5 : if (poDstDriver)
691 : {
692 5 : bTemporaryFile = true;
693 : outputFilename =
694 5 : CPLGenerateTempFilenameSafe(
695 15 : std::string("_").append(defaultLayerName).c_str()) +
696 5 : ".gpkg";
697 5 : format = "GPKG";
698 : }
699 : else
700 0 : format = "MEM";
701 : }
702 :
703 42 : if (!poDstDriver)
704 37 : poDstDriver = poDriverManager->GetDriverByName(format.c_str());
705 42 : if (!poDstDriver)
706 : {
707 0 : ReportError(CE_Failure, CPLE_AppDefined, "Cannot find driver %s",
708 : format.c_str());
709 0 : return false;
710 : }
711 :
712 42 : poNewRetDS.reset(poDstDriver->Create(
713 : outputFilename.c_str(), 0, 0, 0, GDT_Unknown,
714 84 : CPLStringList(poWriteStep->GetCreationOptions()).List()));
715 42 : if (!poNewRetDS)
716 2 : return false;
717 :
718 40 : if (bTemporaryFile)
719 5 : poNewRetDS->MarkSuppressOnClose();
720 :
721 40 : poDstDS = poNewRetDS.get();
722 : }
723 : else
724 : {
725 20 : poDstDriver = poDstDS->GetDriver();
726 : }
727 :
728 60 : outputLayerName = poWriteStep->GetOutputLayerName();
729 60 : if (outputLayerName.empty() && poDstDS->GetLayerCount() > 1)
730 : {
731 3 : ReportError(CE_Failure, CPLE_AppDefined, "--%s must be specified",
732 : GDAL_ARG_NAME_OUTPUT_LAYER);
733 3 : return false;
734 : }
735 :
736 57 : if (poDstDriver && EQUAL(poDstDriver->GetDescription(), "ESRI Shapefile") &&
737 77 : (EQUAL(CPLGetExtensionSafe(poDstDS->GetDescription()).c_str(), "shp") ||
738 60 : EQUAL(CPLGetExtensionSafe(poDstDS->GetDescription()).c_str(),
739 114 : "shz")) &&
740 20 : poDstDS->GetLayerCount() <= 1)
741 : {
742 20 : outputLayerName = CPLGetBasenameSafe(poDstDS->GetDescription());
743 : }
744 57 : if (outputLayerName.empty())
745 29 : outputLayerName = defaultLayerName;
746 :
747 57 : poDstLayer = poDstDS->GetLayerByName(outputLayerName.c_str());
748 57 : if (poDstLayer)
749 : {
750 13 : if (poWriteStep->GetOverwriteLayer())
751 : {
752 5 : int iLayer = -1;
753 5 : const int nLayerCount = poDstDS->GetLayerCount();
754 6 : for (iLayer = 0; iLayer < nLayerCount; iLayer++)
755 : {
756 6 : if (poDstDS->GetLayer(iLayer) == poDstLayer)
757 5 : break;
758 : }
759 :
760 5 : if (iLayer < nLayerCount)
761 : {
762 5 : if (poDstDS->DeleteLayer(iLayer) != OGRERR_NONE)
763 : {
764 2 : ReportError(CE_Failure, CPLE_AppDefined,
765 : "Cannot delete layer '%s'",
766 : outputLayerName.c_str());
767 2 : return false;
768 : }
769 : }
770 3 : poDstLayer = nullptr;
771 : }
772 8 : else if (!poWriteStep->GetAppendLayer())
773 : {
774 3 : ReportError(CE_Failure, CPLE_AppDefined,
775 : "Layer '%s' already exists. Specify the "
776 : "--%s option to overwrite it, or --%s "
777 : "to append to it.",
778 : outputLayerName.c_str(), GDAL_ARG_NAME_OVERWRITE_LAYER,
779 : GDAL_ARG_NAME_APPEND);
780 3 : return false;
781 : }
782 : }
783 44 : else if (poWriteStep->GetAppendLayer() || poWriteStep->GetOverwriteLayer())
784 : {
785 3 : ReportError(CE_Failure, CPLE_AppDefined, "Cannot find layer '%s'",
786 : outputLayerName.c_str());
787 3 : return false;
788 : }
789 :
790 49 : return true;
791 : }
792 :
793 : GDALAlgorithmStepRegistry::~GDALAlgorithmStepRegistry() = default;
794 :
795 : /************************************************************************/
796 : /* GDALPipelineAlgorithm */
797 : /************************************************************************/
798 :
799 334 : GDALPipelineAlgorithm::GDALPipelineAlgorithm()
800 : : GDALAbstractPipelineAlgorithm(
801 : NAME, DESCRIPTION, HELP_URL,
802 334 : ConstructorOptions().SetStandaloneStep(false))
803 : {
804 334 : m_supportsStreamedOutput = true;
805 :
806 334 : AddProgressArg();
807 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_RASTER | GDAL_OF_VECTOR,
808 334 : /* positionalAndRequired = */ false)
809 334 : .SetMinCount(1)
810 334 : .SetMaxCount(INT_MAX)
811 334 : .SetHiddenForCLI();
812 : AddOutputDatasetArg(&m_outputDataset, GDAL_OF_RASTER | GDAL_OF_VECTOR,
813 334 : /* positionalAndRequired = */ false)
814 334 : .SetHiddenForCLI()
815 334 : .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT);
816 : AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
817 334 : /* bGDALGAllowed = */ true)
818 334 : .SetHiddenForCLI();
819 668 : AddArg("pipeline", 0, _("Pipeline string or filename"), &m_pipeline)
820 334 : .SetHiddenForCLI()
821 334 : .SetPositional();
822 :
823 334 : AddOutputStringArg(&m_output).SetHiddenForCLI();
824 334 : AddStdoutArg(&m_stdout);
825 :
826 334 : AllowArbitraryLongNameArgs();
827 :
828 334 : GDALRasterPipelineAlgorithm::RegisterAlgorithms(m_stepRegistry, true);
829 334 : GDALVectorPipelineAlgorithm::RegisterAlgorithms(m_stepRegistry, true);
830 334 : m_stepRegistry.Register<GDALExternalRasterOrVectorAlgorithm>();
831 334 : m_stepRegistry.Register<GDALRasterAsFeaturesAlgorithm>();
832 334 : m_stepRegistry.Register<GDALRasterContourAlgorithm>();
833 334 : m_stepRegistry.Register<GDALRasterFootprintAlgorithm>();
834 334 : m_stepRegistry.Register<GDALRasterPixelInfoAlgorithm>();
835 334 : m_stepRegistry.Register<GDALRasterPolygonizeAlgorithm>();
836 334 : m_stepRegistry.Register<GDALRasterZonalStatsAlgorithm>();
837 334 : m_stepRegistry.Register<GDALVectorGridAlgorithm>();
838 334 : m_stepRegistry.Register<GDALVectorRasterizeAlgorithm>();
839 334 : }
840 :
841 : /************************************************************************/
842 : /* GDALPipelineAlgorithm::GetUsageForCLI() */
843 : /************************************************************************/
844 :
845 : std::string
846 19 : GDALPipelineAlgorithm::GetUsageForCLI(bool shortUsage,
847 : const UsageOptions &usageOptions) const
848 : {
849 19 : UsageOptions stepUsageOptions;
850 19 : stepUsageOptions.isPipelineStep = true;
851 :
852 19 : if (!m_helpDocCategory.empty() && m_helpDocCategory != "main")
853 : {
854 4 : auto alg = GetStepAlg(m_helpDocCategory);
855 2 : if (alg)
856 : {
857 3 : alg->SetCallPath({CPLString(m_helpDocCategory)
858 2 : .replaceAll(RASTER_SUFFIX, "")
859 3 : .replaceAll(VECTOR_SUFFIX, "")});
860 1 : alg->GetArg("help-doc")->Set(true);
861 1 : return alg->GetUsageForCLI(shortUsage, stepUsageOptions);
862 : }
863 : else
864 : {
865 1 : fprintf(stderr, "ERROR: unknown pipeline step '%s'\n",
866 : m_helpDocCategory.c_str());
867 : return CPLSPrintf("ERROR: unknown pipeline step '%s'\n",
868 1 : m_helpDocCategory.c_str());
869 : }
870 : }
871 :
872 17 : UsageOptions usageOptionsMain(usageOptions);
873 17 : usageOptionsMain.isPipelineMain = true;
874 : std::string ret =
875 34 : GDALAlgorithm::GetUsageForCLI(shortUsage, usageOptionsMain);
876 17 : if (shortUsage)
877 15 : return ret;
878 :
879 : ret +=
880 : "\n<PIPELINE> is of the form: read|calc|concat|create|mosaic|stack "
881 : "[READ-OPTIONS] "
882 2 : "( ! <STEP-NAME> [STEP-OPTIONS] )* ! write!info!tile [WRITE-OPTIONS]\n";
883 :
884 2 : if (m_helpDocCategory == "main")
885 : {
886 1 : return ret;
887 : }
888 :
889 1 : ret += '\n';
890 1 : ret += "Example: 'gdal pipeline --progress ! read in.tif ! \\\n";
891 1 : ret += " rasterize --size 256 256 ! buffer 20 ! ";
892 1 : ret += "write out.gpkg --overwrite'\n";
893 1 : ret += '\n';
894 1 : ret += "Potential steps are:\n";
895 :
896 87 : for (const std::string &name : m_stepRegistry.GetNames())
897 : {
898 172 : auto alg = GetStepAlg(name);
899 86 : assert(alg);
900 86 : auto [options, maxOptLen] = alg->GetArgNamesForCLI();
901 86 : stepUsageOptions.maxOptLen =
902 86 : std::max(stepUsageOptions.maxOptLen, maxOptLen);
903 : }
904 :
905 : {
906 1 : ret += '\n';
907 1 : auto alg = std::make_unique<GDALRasterReadAlgorithm>();
908 2 : alg->SetCallPath({alg->GetName()});
909 1 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
910 : }
911 : {
912 1 : ret += '\n';
913 1 : auto alg = std::make_unique<GDALVectorReadAlgorithm>();
914 2 : alg->SetCallPath({alg->GetName()});
915 1 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
916 : }
917 87 : for (const std::string &name : m_stepRegistry.GetNames())
918 : {
919 172 : auto alg = GetStepAlg(name);
920 86 : assert(alg);
921 95 : if (alg->CanBeFirstStep() && !alg->CanBeMiddleStep() &&
922 101 : !alg->IsHidden() &&
923 6 : !STARTS_WITH(name.c_str(), GDALRasterReadAlgorithm::NAME))
924 : {
925 4 : ret += '\n';
926 8 : alg->SetCallPath({name});
927 4 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
928 : }
929 : }
930 87 : for (const std::string &name : m_stepRegistry.GetNames())
931 : {
932 172 : auto alg = GetStepAlg(name);
933 86 : assert(alg);
934 86 : if (alg->CanBeMiddleStep() && !alg->IsHidden())
935 : {
936 72 : ret += '\n';
937 216 : alg->SetCallPath({CPLString(alg->GetName())
938 144 : .replaceAll(RASTER_SUFFIX, "")
939 216 : .replaceAll(VECTOR_SUFFIX, "")});
940 72 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
941 : }
942 : }
943 87 : for (const std::string &name : m_stepRegistry.GetNames())
944 : {
945 172 : auto alg = GetStepAlg(name);
946 86 : assert(alg);
947 99 : if (alg->CanBeLastStep() && !alg->CanBeMiddleStep() &&
948 107 : !alg->IsHidden() &&
949 8 : !STARTS_WITH(name.c_str(), GDALRasterWriteAlgorithm::NAME))
950 : {
951 6 : ret += '\n';
952 12 : alg->SetCallPath({name});
953 6 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
954 : }
955 : }
956 : {
957 1 : ret += '\n';
958 1 : auto alg = std::make_unique<GDALRasterWriteAlgorithm>();
959 2 : alg->SetCallPath({alg->GetName()});
960 1 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
961 : }
962 : {
963 1 : ret += '\n';
964 1 : auto alg = std::make_unique<GDALVectorWriteAlgorithm>();
965 2 : alg->SetCallPath({alg->GetName()});
966 1 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
967 : }
968 :
969 1 : ret += GetUsageForCLIEnd();
970 :
971 1 : return ret;
972 : }
973 :
974 : //! @endcond
|
