Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: GDAL
4 : * Purpose: gdal "pipeline" subcommand
5 : * Author: Even Rouault <even dot rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2024, Even Rouault <even dot rouault at spatialys.com>
9 : *
10 : * SPDX-License-Identifier: MIT
11 : ****************************************************************************/
12 :
13 : //! @cond Doxygen_Suppress
14 :
15 : #include "gdalalg_pipeline.h"
16 : #include "cpl_error.h"
17 : #include "gdal_priv.h"
18 :
19 : #include "gdalalg_external.h"
20 :
21 : #include "gdalalg_raster_read.h"
22 : #include "gdalalg_raster_mosaic.h"
23 : #include "gdalalg_raster_stack.h"
24 : #include "gdalalg_raster_write.h"
25 : #include "gdalalg_raster_zonal_stats.h"
26 :
27 : #include "gdalalg_vector_read.h"
28 : #include "gdalalg_vector_write.h"
29 :
30 : #include "gdalalg_mdim_info.h"
31 : #include "gdalalg_mdim_read.h"
32 : #include "gdalalg_mdim_write.h"
33 :
34 : #include "gdalalg_raster_as_features.h"
35 : #include "gdalalg_raster_compare.h"
36 : #include "gdalalg_raster_contour.h"
37 : #include "gdalalg_raster_footprint.h"
38 : #include "gdalalg_raster_polygonize.h"
39 : #include "gdalalg_raster_info.h"
40 : #include "gdalalg_raster_pixel_info.h"
41 : #include "gdalalg_raster_tile.h"
42 : #include "gdalalg_vector_grid.h"
43 : #include "gdalalg_vector_info.h"
44 : #include "gdalalg_vector_rasterize.h"
45 :
46 : #include <algorithm>
47 : #include <cassert>
48 :
49 : #ifndef _
50 : #define _(x) (x)
51 : #endif
52 :
53 : /************************************************************************/
54 : /* GDALPipelineStepAlgorithm() */
55 : /************************************************************************/
56 :
57 15277 : GDALPipelineStepAlgorithm::GDALPipelineStepAlgorithm(
58 : const std::string &name, const std::string &description,
59 15277 : const std::string &helpURL, const ConstructorOptions &options)
60 : : GDALAlgorithm(name, description, helpURL),
61 15277 : m_standaloneStep(options.standaloneStep), m_constructorOptions(options)
62 : {
63 15277 : }
64 :
65 : /************************************************************************/
66 : /* GDALPipelineStepAlgorithm::AddRasterHiddenInputDatasetArg() */
67 : /************************************************************************/
68 :
69 2297 : void GDALPipelineStepAlgorithm::AddRasterHiddenInputDatasetArg()
70 : {
71 2297 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_RASTER, false)
72 2297 : .SetMinCount(0)
73 2297 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
74 2297 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
75 2297 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
76 2297 : .SetHidden();
77 2297 : }
78 :
79 : /************************************************************************/
80 : /* GDALPipelineStepAlgorithm::AddRasterInputArgs() */
81 : /************************************************************************/
82 :
83 5079 : void GDALPipelineStepAlgorithm::AddRasterInputArgs(
84 : bool openForMixedRasterVector, bool hiddenForCLI)
85 : {
86 5079 : AddInputFormatsArg(&m_inputFormats)
87 : .AddMetadataItem(
88 : GAAMDI_REQUIRED_CAPABILITIES,
89 : openForMixedRasterVector
90 15585 : ? std::vector<std::string>{GDAL_DCAP_RASTER, GDAL_DCAP_VECTOR}
91 14889 : : std::vector<std::string>{GDAL_DCAP_RASTER})
92 5079 : .SetHiddenForCLI(hiddenForCLI)
93 5079 : .SetAvailableInPipelineStep(false);
94 5079 : AddOpenOptionsArg(&m_openOptions)
95 5079 : .SetHiddenForCLI(hiddenForCLI)
96 5079 : .SetAvailableInPipelineStep(false);
97 :
98 5079 : const int nDatasetType = openForMixedRasterVector
99 5079 : ? (GDAL_OF_RASTER | GDAL_OF_VECTOR)
100 : : GDAL_OF_RASTER;
101 : auto &arg =
102 : AddInputDatasetArg(
103 : &m_inputDataset, nDatasetType, false,
104 9681 : m_constructorOptions.inputDatasetHelpMsg.empty() &&
105 4602 : m_constructorOptions.inputDatasetMaxCount == 1
106 4226 : ? CPLSPrintf(
107 : "Input %s dataset",
108 9305 : GDALAlgorithmArgDatasetTypeName(nDatasetType).c_str())
109 13907 : : m_constructorOptions.inputDatasetHelpMsg.c_str())
110 5079 : .SetDatasetInputFlags(m_constructorOptions.inputDatasetInputFlags)
111 5079 : .SetMinCount(m_constructorOptions.inputDatasetRequired ? 1 : 0)
112 5079 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
113 5079 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
114 5079 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
115 5079 : .SetHiddenForCLI(hiddenForCLI)
116 5079 : .SetAvailableInPipelineStep(false);
117 5079 : if (m_constructorOptions.inputDatasetPositional && !hiddenForCLI)
118 4689 : arg.SetPositional();
119 5079 : if (m_constructorOptions.inputDatasetRequired && !hiddenForCLI)
120 4689 : arg.SetRequired();
121 5079 : if (!m_constructorOptions.inputDatasetAlias.empty())
122 897 : arg.AddAlias(m_constructorOptions.inputDatasetAlias);
123 5079 : }
124 :
125 : /************************************************************************/
126 : /* GDALPipelineStepAlgorithm::AddRasterOutputArgs() */
127 : /************************************************************************/
128 :
129 3308 : void GDALPipelineStepAlgorithm::AddRasterOutputArgs(bool hiddenForCLI)
130 : {
131 3308 : m_outputFormatArg =
132 : &(AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
133 3308 : /* bGDALGAllowed = */ true)
134 : .AddMetadataItem(
135 : GAAMDI_REQUIRED_CAPABILITIES,
136 : {GDAL_DCAP_RASTER,
137 13232 : m_constructorOptions.outputFormatCreateCapability.c_str()})
138 3308 : .SetHiddenForCLI(hiddenForCLI))
139 3308 : .SetAvailableInPipelineStep(false);
140 : AddOutputDatasetArg(&m_outputDataset, GDAL_OF_RASTER,
141 3308 : /* positionalAndRequired = */ !hiddenForCLI,
142 3308 : m_constructorOptions.outputDatasetHelpMsg.c_str())
143 3308 : .SetHiddenForCLI(hiddenForCLI)
144 3308 : .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT)
145 3308 : .SetAvailableInPipelineStep(false);
146 3308 : AddCreationOptionsArg(&m_creationOptions)
147 3308 : .SetHiddenForCLI(hiddenForCLI)
148 3308 : .SetAvailableInPipelineStep(false);
149 3308 : constexpr const char *MUTUAL_EXCLUSION_GROUP_OVERWRITE_APPEND =
150 : "overwrite-append";
151 3308 : AddOverwriteArg(&m_overwrite)
152 3308 : .SetHiddenForCLI(hiddenForCLI)
153 6616 : .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_OVERWRITE_APPEND)
154 3308 : .SetAvailableInPipelineStep(false);
155 : AddArg(GDAL_ARG_NAME_APPEND, 0,
156 6616 : _("Append as a subdataset to existing output"), &m_appendRaster)
157 3308 : .SetDefault(false)
158 3308 : .SetHiddenForCLI(hiddenForCLI)
159 6616 : .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_OVERWRITE_APPEND)
160 3308 : .SetAvailableInPipelineStep(false);
161 3308 : }
162 :
163 : /************************************************************************/
164 : /* GDALPipelineStepAlgorithm::AddVectorHiddenInputDatasetArg() */
165 : /************************************************************************/
166 :
167 2207 : void GDALPipelineStepAlgorithm::AddVectorHiddenInputDatasetArg()
168 : {
169 2207 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_VECTOR, false)
170 2207 : .SetMinCount(0)
171 2207 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
172 2207 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
173 2207 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
174 2207 : .SetHidden();
175 2207 : }
176 :
177 : /************************************************************************/
178 : /* GDALPipelineStepAlgorithm::AddVectorInputArgs() */
179 : /************************************************************************/
180 :
181 3053 : void GDALPipelineStepAlgorithm::AddVectorInputArgs(bool hiddenForCLI)
182 : {
183 3053 : AddInputFormatsArg(&m_inputFormats)
184 9159 : .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES, {GDAL_DCAP_VECTOR})
185 3053 : .SetHiddenForCLI(hiddenForCLI)
186 3053 : .SetAvailableInPipelineStep(false);
187 3053 : AddOpenOptionsArg(&m_openOptions)
188 3053 : .SetHiddenForCLI(hiddenForCLI)
189 3053 : .SetAvailableInPipelineStep(false);
190 : auto &datasetArg =
191 : AddInputDatasetArg(
192 : &m_inputDataset, GDAL_OF_VECTOR, false,
193 9027 : m_constructorOptions.inputDatasetHelpMsg.empty() &&
194 2921 : m_constructorOptions.inputDatasetMaxCount == 1
195 : ? "Input vector dataset"
196 5974 : : m_constructorOptions.inputDatasetHelpMsg.c_str())
197 3053 : .SetMinCount(m_constructorOptions.inputDatasetRequired ? 1 : 0)
198 3053 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
199 3053 : .SetDatasetInputFlags(m_constructorOptions.inputDatasetInputFlags)
200 3053 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
201 3053 : .SetHiddenForCLI(hiddenForCLI)
202 3053 : .SetAvailableInPipelineStep(false);
203 3053 : if (!m_constructorOptions.inputDatasetAlias.empty())
204 192 : datasetArg.AddAlias(m_constructorOptions.inputDatasetAlias);
205 3053 : if (!m_constructorOptions.inputDatasetMetaVar.empty())
206 3053 : datasetArg.SetMetaVar(m_constructorOptions.inputDatasetMetaVar);
207 3053 : if (m_constructorOptions.inputDatasetPositional && !hiddenForCLI)
208 2735 : datasetArg.SetPositional();
209 3053 : if (m_constructorOptions.inputDatasetRequired && !hiddenForCLI)
210 2735 : datasetArg.SetRequired();
211 3053 : if (m_constructorOptions.addInputLayerNameArgument)
212 : {
213 : auto &layerArg = AddArg(GDAL_ARG_NAME_INPUT_LAYER, 'l',
214 5392 : _("Input layer name(s)"), &m_inputLayerNames)
215 5392 : .AddAlias("layer")
216 2696 : .SetHiddenForCLI(hiddenForCLI)
217 2696 : .SetAvailableInPipelineStep(false);
218 2696 : SetAutoCompleteFunctionForLayerName(layerArg, datasetArg);
219 : }
220 3053 : }
221 :
222 : /************************************************************************/
223 : /* GDALPipelineStepAlgorithm::AddVectorOutputArgs() */
224 : /************************************************************************/
225 :
226 3495 : void GDALPipelineStepAlgorithm::AddVectorOutputArgs(
227 : bool hiddenForCLI, bool shortNameOutputLayerAllowed)
228 : {
229 : AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
230 3495 : /* bGDALGAllowed = */ true)
231 : .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,
232 13980 : {GDAL_DCAP_VECTOR, GDAL_DCAP_CREATE})
233 3495 : .SetHiddenForCLI(hiddenForCLI)
234 3495 : .SetAvailableInPipelineStep(false);
235 3495 : AddOutputOpenOptionsArg(&m_outputOpenOptions)
236 3495 : .SetHiddenForCLI(hiddenForCLI)
237 3495 : .SetAvailableInPipelineStep(false);
238 : auto &outputDatasetArg =
239 : AddOutputDatasetArg(&m_outputDataset, GDAL_OF_VECTOR,
240 3495 : /* positionalAndRequired = */ false)
241 3495 : .SetHiddenForCLI(hiddenForCLI)
242 3495 : .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT)
243 3495 : .SetAvailableInPipelineStep(false);
244 3495 : if (!hiddenForCLI)
245 3309 : outputDatasetArg.SetPositional();
246 3495 : if (!hiddenForCLI && m_constructorOptions.outputDatasetRequired)
247 3249 : outputDatasetArg.SetRequired();
248 :
249 3495 : AddCreationOptionsArg(&m_creationOptions)
250 3495 : .SetHiddenForCLI(hiddenForCLI)
251 3495 : .SetAvailableInPipelineStep(false);
252 3495 : AddLayerCreationOptionsArg(&m_layerCreationOptions)
253 3495 : .SetHiddenForCLI(hiddenForCLI)
254 3495 : .SetAvailableInPipelineStep(false);
255 3495 : AddOverwriteArg(&m_overwrite)
256 3495 : .SetHiddenForCLI(hiddenForCLI)
257 3495 : .SetAvailableInPipelineStep(false);
258 3495 : GDALInConstructionAlgorithmArg *updateArg = nullptr;
259 3495 : if (m_constructorOptions.addUpdateArgument)
260 : {
261 3432 : updateArg = &AddUpdateArg(&m_update)
262 3432 : .SetHiddenForCLI(hiddenForCLI)
263 3432 : .SetAvailableInPipelineStep(false);
264 : }
265 3495 : if (m_constructorOptions.addOverwriteLayerArgument)
266 : {
267 3432 : AddOverwriteLayerArg(&m_overwriteLayer)
268 3432 : .SetHiddenForCLI(hiddenForCLI)
269 3432 : .SetAvailableInPipelineStep(false);
270 : }
271 3495 : constexpr const char *MUTUAL_EXCLUSION_GROUP_APPEND_UPSERT =
272 : "append-upsert";
273 3495 : if (m_constructorOptions.addAppendLayerArgument)
274 : {
275 3300 : AddAppendLayerArg(&m_appendLayer)
276 3300 : .SetHiddenForCLI(hiddenForCLI)
277 6600 : .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_APPEND_UPSERT)
278 3300 : .SetAvailableInPipelineStep(false);
279 : }
280 3495 : if (m_constructorOptions.addUpsertArgument)
281 : {
282 6170 : AddArg("upsert", 0, _("Upsert features (implies 'append')"), &m_upsert)
283 3085 : .SetHiddenForCLI(hiddenForCLI)
284 6170 : .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_APPEND_UPSERT)
285 :
286 3085 : .SetAvailableInPipelineStep(false)
287 : .AddAction(
288 12 : [updateArg, this]()
289 : {
290 4 : if (m_upsert && updateArg)
291 4 : updateArg->Set(true);
292 6170 : })
293 3085 : .SetCategory(GAAC_ADVANCED);
294 : }
295 3495 : if (m_constructorOptions.addOutputLayerNameArgument)
296 : {
297 3363 : AddOutputLayerNameArg(hiddenForCLI, shortNameOutputLayerAllowed);
298 : }
299 3495 : if (m_constructorOptions.addSkipErrorsArgument)
300 : {
301 : AddArg("skip-errors", 0, _("Skip errors when writing features"),
302 6170 : &m_skipErrors)
303 6170 : .AddHiddenAlias("skip-failures") // For ogr2ogr nostalgic people
304 3085 : .SetAvailableInPipelineStep(false);
305 : }
306 3495 : if (m_constructorOptions.addNoCreateEmptyLayersArgument)
307 : {
308 : AddArg("no-create-empty-layers", 0,
309 : _("Avoid creating layers to which no features will be written"),
310 2360 : &m_noCreateEmptyLayers)
311 1180 : .SetAvailableInPipelineStep(false);
312 : }
313 3495 : }
314 :
315 : /************************************************************************/
316 : /* GDALPipelineStepAlgorithm::AddOutputLayerNameArg() */
317 : /************************************************************************/
318 :
319 3606 : void GDALPipelineStepAlgorithm::AddOutputLayerNameArg(
320 : bool hiddenForCLI, bool shortNameOutputLayerAllowed)
321 : {
322 : AddArg(GDAL_ARG_NAME_OUTPUT_LAYER, shortNameOutputLayerAllowed ? 'l' : 0,
323 : _("Output layer name"),
324 7212 : &m_outputLayerName)
325 7212 : .AddHiddenAlias("nln") // For ogr2ogr nostalgic people
326 3606 : .SetHiddenForCLI(hiddenForCLI)
327 : .SetAvailableInPipelineStep(
328 3606 : m_constructorOptions.outputLayerNameAvailableInPipelineStep);
329 3606 : }
330 :
331 : /************************************************************************/
332 : /* GDALPipelineStepAlgorithm::AddMdimHiddenInputDatasetArg() */
333 : /************************************************************************/
334 :
335 25 : void GDALPipelineStepAlgorithm::AddMdimHiddenInputDatasetArg()
336 : {
337 25 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_MULTIDIM_RASTER, false)
338 25 : .SetMinCount(0)
339 25 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
340 25 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
341 25 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
342 25 : .SetHidden();
343 25 : }
344 :
345 : /************************************************************************/
346 : /* GDALPipelineStepAlgorithm::AddMdimInputArgs() */
347 : /************************************************************************/
348 :
349 377 : void GDALPipelineStepAlgorithm::AddMdimInputArgs(bool openForMixedMdimVector,
350 : bool hiddenForCLI,
351 : bool acceptRaster)
352 : {
353 377 : AddInputFormatsArg(&m_inputFormats)
354 : .AddMetadataItem(
355 : GAAMDI_REQUIRED_CAPABILITIES,
356 : openForMixedMdimVector
357 1131 : ? std::vector<
358 : std::string>{acceptRaster
359 : ? GDAL_ALG_DCAP_RASTER_OR_MULTIDIM_RASTER
360 : : GDAL_DCAP_MULTIDIM_RASTER,
361 0 : GDAL_DCAP_VECTOR}
362 : : std::vector<
363 : std::string>{acceptRaster
364 : ? GDAL_ALG_DCAP_RASTER_OR_MULTIDIM_RASTER
365 1131 : : GDAL_DCAP_MULTIDIM_RASTER})
366 377 : .SetHiddenForCLI(hiddenForCLI)
367 377 : .SetAvailableInPipelineStep(false);
368 377 : AddOpenOptionsArg(&m_openOptions)
369 377 : .SetHiddenForCLI(hiddenForCLI)
370 377 : .SetAvailableInPipelineStep(false);
371 : auto &arg =
372 : AddInputDatasetArg(&m_inputDataset,
373 377 : (acceptRaster ? GDAL_OF_RASTER : 0) |
374 : (openForMixedMdimVector
375 377 : ? (GDAL_OF_MULTIDIM_RASTER | GDAL_OF_VECTOR)
376 : : GDAL_OF_MULTIDIM_RASTER),
377 : false,
378 754 : m_constructorOptions.inputDatasetHelpMsg.c_str())
379 377 : .SetDatasetInputFlags(m_constructorOptions.inputDatasetInputFlags)
380 377 : .SetMinCount(m_constructorOptions.inputDatasetRequired ? 1 : 0)
381 377 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
382 377 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
383 377 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
384 377 : .SetHiddenForCLI(hiddenForCLI)
385 377 : .SetAvailableInPipelineStep(false);
386 377 : if (m_constructorOptions.inputDatasetPositional && !hiddenForCLI)
387 323 : arg.SetPositional();
388 377 : if (m_constructorOptions.inputDatasetRequired && !hiddenForCLI)
389 323 : arg.SetRequired();
390 377 : if (!m_constructorOptions.inputDatasetAlias.empty())
391 57 : arg.AddAlias(m_constructorOptions.inputDatasetAlias);
392 377 : }
393 :
394 : /************************************************************************/
395 : /* GDALPipelineStepAlgorithm::AddMdimOutputArgs() */
396 : /************************************************************************/
397 :
398 245 : void GDALPipelineStepAlgorithm::AddMdimOutputArgs(bool hiddenForCLI)
399 : {
400 245 : m_outputFormatArg =
401 : &(AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
402 245 : /* bGDALGAllowed = */ true)
403 : .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,
404 735 : {GDAL_DCAP_CREATE_MULTIDIMENSIONAL})
405 245 : .SetHiddenForCLI(hiddenForCLI))
406 245 : .SetAvailableInPipelineStep(false);
407 : AddOutputDatasetArg(&m_outputDataset, GDAL_OF_MULTIDIM_RASTER,
408 245 : /* positionalAndRequired = */ !hiddenForCLI,
409 245 : m_constructorOptions.outputDatasetHelpMsg.c_str())
410 245 : .SetHiddenForCLI(hiddenForCLI)
411 245 : .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT)
412 245 : .SetAvailableInPipelineStep(false);
413 245 : AddCreationOptionsArg(&m_creationOptions)
414 245 : .SetHiddenForCLI(hiddenForCLI)
415 245 : .SetAvailableInPipelineStep(false);
416 245 : AddOverwriteArg(&m_overwrite)
417 245 : .SetHiddenForCLI(hiddenForCLI)
418 245 : .SetAvailableInPipelineStep(false);
419 245 : }
420 :
421 : /************************************************************************/
422 : /* GDALPipelineStepAlgorithm::RunImpl() */
423 : /************************************************************************/
424 :
425 3112 : bool GDALPipelineStepAlgorithm::RunImpl(GDALProgressFunc pfnProgress,
426 : void *pProgressData)
427 : {
428 3112 : if (m_standaloneStep)
429 : {
430 1296 : std::unique_ptr<GDALPipelineStepAlgorithm> readAlg;
431 1296 : if (GetInputType() == GDAL_OF_RASTER)
432 800 : readAlg = std::make_unique<GDALRasterReadAlgorithm>();
433 496 : else if (GetInputType() == GDAL_OF_MULTIDIM_RASTER)
434 22 : readAlg = std::make_unique<GDALMdimReadAlgorithm>();
435 : else
436 474 : readAlg = std::make_unique<GDALVectorReadAlgorithm>();
437 10842 : for (auto &arg : readAlg->GetArgs())
438 : {
439 9546 : auto stepArg = GetArg(arg->GetName());
440 10859 : if (stepArg && stepArg->IsExplicitlySet() &&
441 1313 : stepArg->GetType() == arg->GetType())
442 : {
443 1310 : arg->SetSkipIfAlreadySet(true);
444 1310 : arg->SetFrom(*stepArg);
445 : }
446 : }
447 :
448 0 : std::unique_ptr<GDALPipelineStepAlgorithm> writeAlg;
449 1296 : if (GetOutputType() == GDAL_OF_RASTER)
450 : {
451 571 : if (GetName() == GDALRasterInfoAlgorithm::NAME)
452 26 : writeAlg = std::make_unique<GDALRasterInfoAlgorithm>();
453 545 : else if (GetName() == GDALRasterCompareAlgorithm::NAME)
454 0 : writeAlg = std::make_unique<GDALRasterCompareAlgorithm>();
455 : else
456 545 : writeAlg = std::make_unique<GDALRasterWriteAlgorithm>();
457 : }
458 725 : else if (GetOutputType() == GDAL_OF_MULTIDIM_RASTER)
459 : {
460 13 : if (GetName() == GDALMdimInfoAlgorithm::NAME)
461 10 : writeAlg = std::make_unique<GDALMdimInfoAlgorithm>();
462 : else
463 3 : writeAlg = std::make_unique<GDALMdimWriteAlgorithm>();
464 : }
465 : else
466 : {
467 712 : if (GetName() == GDALVectorInfoAlgorithm::NAME)
468 27 : writeAlg = std::make_unique<GDALVectorInfoAlgorithm>();
469 : else
470 685 : writeAlg = std::make_unique<GDALVectorWriteAlgorithm>();
471 : }
472 21039 : for (auto &arg : writeAlg->GetArgs())
473 : {
474 19743 : auto stepArg = GetArg(arg->GetName());
475 23027 : if (stepArg && stepArg->IsExplicitlySet() &&
476 3284 : stepArg->GetType() == arg->GetType())
477 : {
478 3278 : arg->SetSkipIfAlreadySet(true);
479 3278 : arg->SetFrom(*stepArg);
480 : }
481 : }
482 :
483 1296 : const bool bIsStreaming = m_format == "stream";
484 :
485 : // Already checked by GDALAlgorithm::Run()
486 1296 : CPLAssert(!m_executionForStreamOutput || bIsStreaming);
487 :
488 1296 : bool ret = false;
489 1352 : if (!m_outputVRTCompatible &&
490 112 : (EQUAL(m_format.c_str(), "VRT") ||
491 56 : (m_format.empty() &&
492 1297 : EQUAL(CPLGetExtensionSafe(m_outputDataset.GetName().c_str())
493 : .c_str(),
494 : "VRT"))))
495 : {
496 0 : ReportError(CE_Failure, CPLE_NotSupported,
497 : "VRT output is not supported. Consider using the "
498 : "GDALG driver instead (files with .gdalg.json "
499 : "extension)");
500 : }
501 1296 : else if (readAlg->Run())
502 : {
503 1292 : auto outputArg = GetArg(GDAL_ARG_NAME_OUTPUT);
504 : const bool bOutputSpecified =
505 2297 : outputArg && outputArg->IsExplicitlySet() &&
506 1005 : (outputArg->GetType() == GAAT_DATASET ||
507 3 : outputArg->GetType() == GAAT_DATASET_LIST);
508 :
509 1292 : m_inputDataset.clear();
510 1292 : m_inputDataset.resize(1);
511 1292 : m_inputDataset[0].Set(readAlg->m_outputDataset.GetDatasetRef());
512 1292 : if (bOutputSpecified)
513 999 : m_outputDataset.Set(nullptr);
514 :
515 : std::unique_ptr<void, decltype(&GDALDestroyScaledProgress)>
516 2584 : pScaledData(nullptr, GDALDestroyScaledProgress);
517 :
518 : const bool bCanHandleNextStep =
519 1292 : !bIsStreaming && CanHandleNextStep(writeAlg.get());
520 :
521 1292 : GDALPipelineStepRunContext stepCtxt;
522 1292 : if (pfnProgress && GetName() == GDALRasterCompareAlgorithm::NAME)
523 : {
524 11 : stepCtxt.m_pfnProgress = pfnProgress;
525 11 : stepCtxt.m_pProgressData = pProgressData;
526 : }
527 1335 : else if (pfnProgress &&
528 54 : (bCanHandleNextStep || !IsNativelyStreamingCompatible()))
529 : {
530 52 : pScaledData.reset(GDALCreateScaledProgress(
531 22 : 0.0, bIsStreaming || bCanHandleNextStep ? 1.0 : 0.5,
532 : pfnProgress, pProgressData));
533 30 : stepCtxt.m_pfnProgress =
534 30 : pScaledData ? GDALScaledProgress : nullptr;
535 30 : stepCtxt.m_pProgressData = pScaledData.get();
536 : }
537 :
538 1292 : if (bCanHandleNextStep)
539 52 : stepCtxt.m_poNextUsableStep = writeAlg.get();
540 1292 : if (RunPreStepPipelineValidations() && RunStep(stepCtxt))
541 : {
542 1178 : if (bCanHandleNextStep || !bOutputSpecified)
543 : {
544 331 : ret = true;
545 : }
546 : else
547 : {
548 847 : writeAlg->m_outputVRTCompatible = m_outputVRTCompatible;
549 847 : writeAlg->m_quiet = m_quiet;
550 :
551 1694 : std::vector<GDALArgDatasetValue> inputDataset(1);
552 847 : inputDataset[0].Set(m_outputDataset.GetDatasetRef());
553 847 : auto inputArg = writeAlg->GetArg(GDAL_ARG_NAME_INPUT);
554 847 : CPLAssert(inputArg);
555 847 : inputArg->Set(std::move(inputDataset));
556 :
557 847 : if (pfnProgress)
558 : {
559 41 : pScaledData.reset(GDALCreateScaledProgress(
560 41 : IsNativelyStreamingCompatible() ? 0.0 : 0.5, 1.0,
561 : pfnProgress, pProgressData));
562 : }
563 847 : stepCtxt.m_pfnProgress =
564 847 : pScaledData ? GDALScaledProgress : nullptr;
565 847 : stepCtxt.m_pProgressData = pScaledData.get();
566 1694 : if (writeAlg->ValidateArguments() &&
567 847 : writeAlg->RunStep(stepCtxt))
568 : {
569 820 : if (pfnProgress)
570 40 : pfnProgress(1.0, "", pProgressData);
571 :
572 820 : m_outputDataset.Set(
573 820 : writeAlg->m_outputDataset.GetDatasetRef());
574 820 : ret = true;
575 : }
576 : }
577 : }
578 : }
579 :
580 1296 : return ret;
581 : }
582 : else
583 : {
584 1816 : GDALPipelineStepRunContext stepCtxt;
585 1816 : stepCtxt.m_pfnProgress = pfnProgress;
586 1816 : stepCtxt.m_pProgressData = pProgressData;
587 1816 : return RunPreStepPipelineValidations() && RunStep(stepCtxt);
588 : }
589 : }
590 :
591 : /************************************************************************/
592 : /* SetInputDataset() */
593 : /************************************************************************/
594 :
595 27 : void GDALPipelineStepAlgorithm::SetInputDataset(GDALDataset *poDS)
596 : {
597 27 : auto arg = GetArg(GDAL_ARG_NAME_INPUT);
598 27 : if (arg)
599 : {
600 27 : auto &val = arg->Get<std::vector<GDALArgDatasetValue>>();
601 27 : val.resize(1);
602 27 : val[0].Set(poDS);
603 27 : arg->NotifyValueSet();
604 27 : arg->SetSkipIfAlreadySet();
605 : }
606 27 : }
607 :
608 : /************************************************************************/
609 : /* ProcessGDALGOutput() */
610 : /************************************************************************/
611 :
612 : GDALAlgorithm::ProcessGDALGOutputRet
613 4295 : GDALPipelineStepAlgorithm::ProcessGDALGOutput()
614 : {
615 4295 : if (m_standaloneStep)
616 : {
617 2395 : return GDALAlgorithm::ProcessGDALGOutput();
618 : }
619 : else
620 : {
621 : // GDALAbstractPipelineAlgorithm<StepAlgorithm>::RunStep() might
622 : // actually detect a GDALG output request and process it.
623 1900 : return GDALAlgorithm::ProcessGDALGOutputRet::NOT_GDALG;
624 : }
625 : }
626 :
627 : /************************************************************************/
628 : /* GDALPipelineStepAlgorithm::CheckSafeForStreamOutput() */
629 : /************************************************************************/
630 :
631 98 : bool GDALPipelineStepAlgorithm::CheckSafeForStreamOutput()
632 : {
633 98 : if (m_standaloneStep)
634 : {
635 50 : return GDALAlgorithm::CheckSafeForStreamOutput();
636 : }
637 : else
638 : {
639 : // The check is actually done in
640 : // GDALAbstractPipelineAlgorithm<StepAlgorithm>::RunStep()
641 : // so return true for now.
642 48 : return true;
643 : }
644 : }
645 :
646 : /************************************************************************/
647 : /* GDALPipelineStepAlgorithm::Finalize() */
648 : /************************************************************************/
649 :
650 1616 : bool GDALPipelineStepAlgorithm::Finalize()
651 : {
652 1616 : bool ret = GDALAlgorithm::Finalize();
653 3012 : for (auto &argValue : m_inputDataset)
654 1396 : ret = argValue.Close() && ret;
655 1616 : ret = m_outputDataset.Close() && ret;
656 1616 : return ret;
657 : }
658 :
659 : /************************************************************************/
660 : /* CreateDatasetSingleOutputLayerIfNeeded() */
661 : /************************************************************************/
662 :
663 68 : bool GDALPipelineStepAlgorithm::CreateDatasetSingleOutputLayerIfNeeded(
664 : GDALPipelineStepRunContext &ctxt, const std::string &defaultLayerName,
665 : GDALDataset *&poDstDS, bool &bTemporaryFile,
666 : std::unique_ptr<GDALDataset> &poNewRetDS, std::string &outputLayerName,
667 : OGRLayer *&poDstLayer)
668 : {
669 68 : auto poWriteStep = ctxt.m_poNextUsableStep ? ctxt.m_poNextUsableStep : this;
670 136 : std::string outputFilename = poWriteStep->GetOutputDataset().GetName();
671 :
672 68 : poDstDS = poWriteStep->GetOutputDataset().GetDatasetRef();
673 68 : poNewRetDS.reset();
674 68 : bTemporaryFile = false;
675 68 : poDstLayer = nullptr;
676 68 : GDALDriver *poDstDriver = nullptr;
677 68 : if (!poDstDS)
678 : {
679 48 : auto poDriverManager = GetGDALDriverManager();
680 48 : std::string format = poWriteStep->GetOutputFormat();
681 48 : if (m_standaloneStep || (ctxt.m_poNextUsableStep && format.empty()))
682 : {
683 41 : if (format.empty())
684 : {
685 : const auto aosFormats =
686 : CPLStringList(GDALGetOutputDriversForDatasetName(
687 : outputFilename.c_str(), GDAL_OF_VECTOR,
688 : /* bSingleMatch = */ true,
689 25 : /* bWarn = */ true));
690 25 : if (aosFormats.size() != 1)
691 : {
692 2 : ReportError(CE_Failure, CPLE_AppDefined,
693 : "Cannot guess driver for %s",
694 : outputFilename.c_str());
695 2 : return false;
696 : }
697 23 : format = aosFormats[0];
698 : }
699 : }
700 7 : else if (!ctxt.m_poNextUsableStep)
701 : {
702 5 : poDstDriver = poDriverManager->GetDriverByName("GPKG");
703 5 : if (poDstDriver)
704 : {
705 5 : bTemporaryFile = true;
706 : outputFilename =
707 5 : CPLGenerateTempFilenameSafe(
708 15 : std::string("_").append(defaultLayerName).c_str()) +
709 5 : ".gpkg";
710 5 : format = "GPKG";
711 : }
712 : else
713 0 : format = "MEM";
714 : }
715 :
716 46 : if (!poDstDriver)
717 41 : poDstDriver = poDriverManager->GetDriverByName(format.c_str());
718 46 : if (!poDstDriver)
719 : {
720 0 : ReportError(CE_Failure, CPLE_AppDefined, "Cannot find driver %s",
721 : format.c_str());
722 0 : return false;
723 : }
724 :
725 46 : poNewRetDS.reset(poDstDriver->Create(
726 : outputFilename.c_str(), 0, 0, 0, GDT_Unknown,
727 92 : CPLStringList(poWriteStep->GetCreationOptions()).List()));
728 46 : if (!poNewRetDS)
729 2 : return false;
730 :
731 44 : if (bTemporaryFile)
732 5 : poNewRetDS->MarkSuppressOnClose();
733 :
734 44 : poDstDS = poNewRetDS.get();
735 : }
736 : else
737 : {
738 20 : poDstDriver = poDstDS->GetDriver();
739 : }
740 :
741 64 : outputLayerName = poWriteStep->GetOutputLayerName();
742 64 : if (outputLayerName.empty() && poDstDS->GetLayerCount() > 1)
743 : {
744 3 : ReportError(CE_Failure, CPLE_AppDefined, "--%s must be specified",
745 : GDAL_ARG_NAME_OUTPUT_LAYER);
746 3 : return false;
747 : }
748 :
749 61 : if (poDstDriver && EQUAL(poDstDriver->GetDescription(), "ESRI Shapefile") &&
750 81 : (EQUAL(CPLGetExtensionSafe(poDstDS->GetDescription()).c_str(), "shp") ||
751 64 : EQUAL(CPLGetExtensionSafe(poDstDS->GetDescription()).c_str(),
752 122 : "shz")) &&
753 20 : poDstDS->GetLayerCount() <= 1)
754 : {
755 20 : outputLayerName = CPLGetBasenameSafe(poDstDS->GetDescription());
756 : }
757 61 : if (outputLayerName.empty())
758 33 : outputLayerName = defaultLayerName;
759 :
760 61 : poDstLayer = poDstDS->GetLayerByName(outputLayerName.c_str());
761 61 : if (poDstLayer)
762 : {
763 13 : if (poWriteStep->GetOverwriteLayer())
764 : {
765 5 : int iLayer = -1;
766 5 : const int nLayerCount = poDstDS->GetLayerCount();
767 6 : for (iLayer = 0; iLayer < nLayerCount; iLayer++)
768 : {
769 6 : if (poDstDS->GetLayer(iLayer) == poDstLayer)
770 5 : break;
771 : }
772 :
773 5 : if (iLayer < nLayerCount)
774 : {
775 5 : if (poDstDS->DeleteLayer(iLayer) != OGRERR_NONE)
776 : {
777 2 : ReportError(CE_Failure, CPLE_AppDefined,
778 : "Cannot delete layer '%s'",
779 : outputLayerName.c_str());
780 2 : return false;
781 : }
782 : }
783 3 : poDstLayer = nullptr;
784 : }
785 8 : else if (!poWriteStep->GetAppendLayer())
786 : {
787 3 : ReportError(CE_Failure, CPLE_AppDefined,
788 : "Layer '%s' already exists. Specify the "
789 : "--%s option to overwrite it, or --%s "
790 : "to append to it.",
791 : outputLayerName.c_str(), GDAL_ARG_NAME_OVERWRITE_LAYER,
792 : GDAL_ARG_NAME_APPEND);
793 3 : return false;
794 : }
795 : }
796 48 : else if (poWriteStep->GetAppendLayer() || poWriteStep->GetOverwriteLayer())
797 : {
798 3 : ReportError(CE_Failure, CPLE_AppDefined, "Cannot find layer '%s'",
799 : outputLayerName.c_str());
800 3 : return false;
801 : }
802 :
803 53 : return true;
804 : }
805 :
806 : GDALAlgorithmStepRegistry::~GDALAlgorithmStepRegistry() = default;
807 :
808 : /************************************************************************/
809 : /* GDALPipelineAlgorithm */
810 : /************************************************************************/
811 :
812 338 : GDALPipelineAlgorithm::GDALPipelineAlgorithm()
813 : : GDALAbstractPipelineAlgorithm(
814 : NAME, DESCRIPTION, HELP_URL,
815 338 : ConstructorOptions().SetStandaloneStep(false))
816 : {
817 338 : m_supportsStreamedOutput = true;
818 :
819 338 : AddProgressArg();
820 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_RASTER | GDAL_OF_VECTOR,
821 338 : /* positionalAndRequired = */ false)
822 338 : .SetMinCount(1)
823 338 : .SetMaxCount(INT_MAX)
824 338 : .SetHiddenForCLI();
825 : AddOutputDatasetArg(&m_outputDataset, GDAL_OF_RASTER | GDAL_OF_VECTOR,
826 338 : /* positionalAndRequired = */ false)
827 338 : .SetHiddenForCLI()
828 338 : .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT);
829 : AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
830 338 : /* bGDALGAllowed = */ true)
831 338 : .SetHiddenForCLI();
832 676 : AddArg("pipeline", 0, _("Pipeline string or filename"), &m_pipeline)
833 338 : .SetHiddenForCLI()
834 338 : .SetPositional();
835 :
836 338 : AddOutputStringArg(&m_output).SetHiddenForCLI();
837 338 : AddStdoutArg(&m_stdout);
838 :
839 338 : AllowArbitraryLongNameArgs();
840 :
841 338 : GDALRasterPipelineAlgorithm::RegisterAlgorithms(m_stepRegistry, true);
842 338 : GDALVectorPipelineAlgorithm::RegisterAlgorithms(m_stepRegistry, true);
843 338 : m_stepRegistry.Register<GDALExternalRasterOrVectorAlgorithm>();
844 338 : m_stepRegistry.Register<GDALRasterAsFeaturesAlgorithm>();
845 338 : m_stepRegistry.Register<GDALRasterContourAlgorithm>();
846 338 : m_stepRegistry.Register<GDALRasterFootprintAlgorithm>();
847 338 : m_stepRegistry.Register<GDALRasterPixelInfoAlgorithm>();
848 338 : m_stepRegistry.Register<GDALRasterPolygonizeAlgorithm>();
849 338 : m_stepRegistry.Register<GDALRasterZonalStatsAlgorithm>();
850 338 : m_stepRegistry.Register<GDALVectorGridAlgorithm>();
851 338 : m_stepRegistry.Register<GDALVectorRasterizeAlgorithm>();
852 338 : }
853 :
854 : /************************************************************************/
855 : /* GDALPipelineAlgorithm::GetUsageForCLI() */
856 : /************************************************************************/
857 :
858 : std::string
859 19 : GDALPipelineAlgorithm::GetUsageForCLI(bool shortUsage,
860 : const UsageOptions &usageOptions) const
861 : {
862 19 : UsageOptions stepUsageOptions;
863 19 : stepUsageOptions.isPipelineStep = true;
864 :
865 19 : if (!m_helpDocCategory.empty() && m_helpDocCategory != "main")
866 : {
867 4 : auto alg = GetStepAlg(m_helpDocCategory);
868 2 : if (alg)
869 : {
870 3 : alg->SetCallPath({CPLString(m_helpDocCategory)
871 2 : .replaceAll(RASTER_SUFFIX, "")
872 3 : .replaceAll(VECTOR_SUFFIX, "")});
873 1 : alg->GetArg("help-doc")->Set(true);
874 1 : return alg->GetUsageForCLI(shortUsage, stepUsageOptions);
875 : }
876 : else
877 : {
878 1 : fprintf(stderr, "ERROR: unknown pipeline step '%s'\n",
879 : m_helpDocCategory.c_str());
880 : return CPLSPrintf("ERROR: unknown pipeline step '%s'\n",
881 1 : m_helpDocCategory.c_str());
882 : }
883 : }
884 :
885 17 : UsageOptions usageOptionsMain(usageOptions);
886 17 : usageOptionsMain.isPipelineMain = true;
887 : std::string ret =
888 34 : GDALAlgorithm::GetUsageForCLI(shortUsage, usageOptionsMain);
889 17 : if (shortUsage)
890 15 : return ret;
891 :
892 : ret +=
893 : "\n<PIPELINE> is of the form: read|calc|concat|create|mosaic|stack "
894 : "[READ-OPTIONS] "
895 2 : "( ! <STEP-NAME> [STEP-OPTIONS] )* ! write!info!tile [WRITE-OPTIONS]\n";
896 :
897 2 : if (m_helpDocCategory == "main")
898 : {
899 1 : return ret;
900 : }
901 :
902 1 : ret += '\n';
903 1 : ret += "Example: 'gdal pipeline --progress ! read in.tif ! \\\n";
904 1 : ret += " rasterize --size 256 256 ! buffer 20 ! ";
905 1 : ret += "write out.gpkg --overwrite'\n";
906 1 : ret += '\n';
907 1 : ret += "Potential steps are:\n";
908 :
909 88 : for (const std::string &name : m_stepRegistry.GetNames())
910 : {
911 174 : auto alg = GetStepAlg(name);
912 87 : assert(alg);
913 87 : auto [options, maxOptLen] = alg->GetArgNamesForCLI();
914 87 : stepUsageOptions.maxOptLen =
915 87 : std::max(stepUsageOptions.maxOptLen, maxOptLen);
916 : }
917 :
918 : {
919 1 : ret += '\n';
920 1 : auto alg = std::make_unique<GDALRasterReadAlgorithm>();
921 2 : alg->SetCallPath({alg->GetName()});
922 1 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
923 : }
924 : {
925 1 : ret += '\n';
926 1 : auto alg = std::make_unique<GDALVectorReadAlgorithm>();
927 2 : alg->SetCallPath({alg->GetName()});
928 1 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
929 : }
930 88 : for (const std::string &name : m_stepRegistry.GetNames())
931 : {
932 174 : auto alg = GetStepAlg(name);
933 87 : assert(alg);
934 96 : if (alg->CanBeFirstStep() && !alg->CanBeMiddleStep() &&
935 101 : !alg->IsHidden() &&
936 5 : !STARTS_WITH(name.c_str(), GDALRasterReadAlgorithm::NAME))
937 : {
938 3 : ret += '\n';
939 6 : alg->SetCallPath({name});
940 3 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
941 : }
942 : }
943 88 : for (const std::string &name : m_stepRegistry.GetNames())
944 : {
945 174 : auto alg = GetStepAlg(name);
946 87 : assert(alg);
947 87 : if (alg->CanBeMiddleStep() && !alg->IsHidden())
948 : {
949 74 : ret += '\n';
950 222 : alg->SetCallPath({CPLString(alg->GetName())
951 148 : .replaceAll(RASTER_SUFFIX, "")
952 222 : .replaceAll(VECTOR_SUFFIX, "")});
953 74 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
954 : }
955 : }
956 88 : for (const std::string &name : m_stepRegistry.GetNames())
957 : {
958 174 : auto alg = GetStepAlg(name);
959 87 : assert(alg);
960 100 : if (alg->CanBeLastStep() && !alg->CanBeMiddleStep() &&
961 108 : !alg->IsHidden() &&
962 8 : !STARTS_WITH(name.c_str(), GDALRasterWriteAlgorithm::NAME))
963 : {
964 6 : ret += '\n';
965 12 : alg->SetCallPath({name});
966 6 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
967 : }
968 : }
969 : {
970 1 : ret += '\n';
971 1 : auto alg = std::make_unique<GDALRasterWriteAlgorithm>();
972 2 : alg->SetCallPath({alg->GetName()});
973 1 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
974 : }
975 : {
976 1 : ret += '\n';
977 1 : auto alg = std::make_unique<GDALVectorWriteAlgorithm>();
978 2 : alg->SetCallPath({alg->GetName()});
979 1 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
980 : }
981 :
982 1 : ret += GetUsageForCLIEnd();
983 :
984 1 : return ret;
985 : }
986 :
987 : //! @endcond
|
