Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: GDAL
4 : * Purpose: gdal "pipeline" subcommand
5 : * Author: Even Rouault <even dot rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2024, Even Rouault <even dot rouault at spatialys.com>
9 : *
10 : * SPDX-License-Identifier: MIT
11 : ****************************************************************************/
12 :
13 : //! @cond Doxygen_Suppress
14 :
15 : #include "gdalalg_pipeline.h"
16 : #include "cpl_error.h"
17 : #include "gdal_priv.h"
18 :
19 : #include "gdalalg_external.h"
20 :
21 : #include "gdalalg_raster_read.h"
22 : #include "gdalalg_raster_mosaic.h"
23 : #include "gdalalg_raster_stack.h"
24 : #include "gdalalg_raster_write.h"
25 : #include "gdalalg_raster_zonal_stats.h"
26 :
27 : #include "gdalalg_vector_read.h"
28 : #include "gdalalg_vector_write.h"
29 :
30 : #include "gdalalg_mdim_info.h"
31 : #include "gdalalg_mdim_read.h"
32 : #include "gdalalg_mdim_write.h"
33 :
34 : #include "gdalalg_raster_as_features.h"
35 : #include "gdalalg_raster_compare.h"
36 : #include "gdalalg_raster_contour.h"
37 : #include "gdalalg_raster_footprint.h"
38 : #include "gdalalg_raster_polygonize.h"
39 : #include "gdalalg_raster_info.h"
40 : #include "gdalalg_raster_pixel_info.h"
41 : #include "gdalalg_raster_tile.h"
42 : #include "gdalalg_vector_grid.h"
43 : #include "gdalalg_vector_info.h"
44 : #include "gdalalg_vector_rasterize.h"
45 :
46 : #include <algorithm>
47 : #include <cassert>
48 :
49 : #ifndef _
50 : #define _(x) (x)
51 : #endif
52 :
53 : /************************************************************************/
54 : /* GDALPipelineStepAlgorithm() */
55 : /************************************************************************/
56 :
57 15282 : GDALPipelineStepAlgorithm::GDALPipelineStepAlgorithm(
58 : const std::string &name, const std::string &description,
59 15282 : const std::string &helpURL, const ConstructorOptions &options)
60 : : GDALAlgorithm(name, description, helpURL),
61 15282 : m_standaloneStep(options.standaloneStep), m_constructorOptions(options)
62 : {
63 15282 : }
64 :
65 : /************************************************************************/
66 : /* GDALPipelineStepAlgorithm::AddRasterHiddenInputDatasetArg() */
67 : /************************************************************************/
68 :
69 2296 : void GDALPipelineStepAlgorithm::AddRasterHiddenInputDatasetArg()
70 : {
71 2296 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_RASTER, false)
72 2296 : .SetMinCount(0)
73 2296 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
74 2296 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
75 2296 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
76 2296 : .SetHidden();
77 2296 : }
78 :
79 : /************************************************************************/
80 : /* GDALPipelineStepAlgorithm::AddRasterInputArgs() */
81 : /************************************************************************/
82 :
83 5079 : void GDALPipelineStepAlgorithm::AddRasterInputArgs(
84 : bool openForMixedRasterVector, bool hiddenForCLI)
85 : {
86 5079 : AddInputFormatsArg(&m_inputFormats)
87 : .AddMetadataItem(
88 : GAAMDI_REQUIRED_CAPABILITIES,
89 : openForMixedRasterVector
90 15585 : ? std::vector<std::string>{GDAL_DCAP_RASTER, GDAL_DCAP_VECTOR}
91 14889 : : std::vector<std::string>{GDAL_DCAP_RASTER})
92 5079 : .SetHiddenForCLI(hiddenForCLI)
93 5079 : .SetAvailableInPipelineStep(false);
94 5079 : AddOpenOptionsArg(&m_openOptions)
95 5079 : .SetHiddenForCLI(hiddenForCLI)
96 5079 : .SetAvailableInPipelineStep(false);
97 :
98 5079 : const int nDatasetType = openForMixedRasterVector
99 5079 : ? (GDAL_OF_RASTER | GDAL_OF_VECTOR)
100 : : GDAL_OF_RASTER;
101 : auto &arg =
102 : AddInputDatasetArg(
103 : &m_inputDataset, nDatasetType, false,
104 9681 : m_constructorOptions.inputDatasetHelpMsg.empty() &&
105 4602 : m_constructorOptions.inputDatasetMaxCount == 1
106 4226 : ? CPLSPrintf(
107 : "Input %s dataset",
108 9305 : GDALAlgorithmArgDatasetTypeName(nDatasetType).c_str())
109 13907 : : m_constructorOptions.inputDatasetHelpMsg.c_str())
110 5079 : .SetDatasetInputFlags(m_constructorOptions.inputDatasetInputFlags)
111 5079 : .SetMinCount(m_constructorOptions.inputDatasetRequired ? 1 : 0)
112 5079 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
113 5079 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
114 5079 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
115 5079 : .SetHiddenForCLI(hiddenForCLI)
116 5079 : .SetAvailableInPipelineStep(false);
117 5079 : if (m_constructorOptions.inputDatasetPositional && !hiddenForCLI)
118 4689 : arg.SetPositional();
119 5079 : if (m_constructorOptions.inputDatasetRequired && !hiddenForCLI)
120 4689 : arg.SetRequired();
121 5079 : if (!m_constructorOptions.inputDatasetAlias.empty())
122 897 : arg.AddAlias(m_constructorOptions.inputDatasetAlias);
123 5079 : }
124 :
125 : /************************************************************************/
126 : /* GDALPipelineStepAlgorithm::AddRasterOutputArgs() */
127 : /************************************************************************/
128 :
129 3308 : void GDALPipelineStepAlgorithm::AddRasterOutputArgs(bool hiddenForCLI)
130 : {
131 3308 : m_outputFormatArg =
132 : &(AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
133 3308 : /* bGDALGAllowed = */ true)
134 : .AddMetadataItem(
135 : GAAMDI_REQUIRED_CAPABILITIES,
136 : {GDAL_DCAP_RASTER,
137 13232 : m_constructorOptions.outputFormatCreateCapability.c_str()})
138 3308 : .SetHiddenForCLI(hiddenForCLI))
139 3308 : .SetAvailableInPipelineStep(false);
140 : AddOutputDatasetArg(&m_outputDataset, GDAL_OF_RASTER,
141 3308 : /* positionalAndRequired = */ !hiddenForCLI,
142 3308 : m_constructorOptions.outputDatasetHelpMsg.c_str())
143 3308 : .SetHiddenForCLI(hiddenForCLI)
144 3308 : .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT)
145 3308 : .SetAvailableInPipelineStep(false);
146 3308 : AddCreationOptionsArg(&m_creationOptions)
147 3308 : .SetHiddenForCLI(hiddenForCLI)
148 3308 : .SetAvailableInPipelineStep(false);
149 3308 : constexpr const char *MUTUAL_EXCLUSION_GROUP_OVERWRITE_APPEND =
150 : "overwrite-append";
151 3308 : AddOverwriteArg(&m_overwrite)
152 3308 : .SetHiddenForCLI(hiddenForCLI)
153 6616 : .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_OVERWRITE_APPEND)
154 3308 : .SetAvailableInPipelineStep(false);
155 : AddArg(GDAL_ARG_NAME_APPEND, 0,
156 6616 : _("Append as a subdataset to existing output"), &m_appendRaster)
157 3308 : .SetDefault(false)
158 3308 : .SetHiddenForCLI(hiddenForCLI)
159 6616 : .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_OVERWRITE_APPEND)
160 3308 : .SetAvailableInPipelineStep(false);
161 3308 : }
162 :
163 : /************************************************************************/
164 : /* GDALPipelineStepAlgorithm::AddVectorHiddenInputDatasetArg() */
165 : /************************************************************************/
166 :
167 2207 : void GDALPipelineStepAlgorithm::AddVectorHiddenInputDatasetArg()
168 : {
169 2207 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_VECTOR, false)
170 2207 : .SetMinCount(0)
171 2207 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
172 2207 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
173 2207 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
174 2207 : .SetHidden();
175 2207 : }
176 :
177 : /************************************************************************/
178 : /* GDALPipelineStepAlgorithm::AddVectorInputArgs() */
179 : /************************************************************************/
180 :
181 3053 : void GDALPipelineStepAlgorithm::AddVectorInputArgs(bool hiddenForCLI)
182 : {
183 3053 : AddInputFormatsArg(&m_inputFormats)
184 9159 : .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES, {GDAL_DCAP_VECTOR})
185 3053 : .SetHiddenForCLI(hiddenForCLI)
186 3053 : .SetAvailableInPipelineStep(false);
187 3053 : AddOpenOptionsArg(&m_openOptions)
188 3053 : .SetHiddenForCLI(hiddenForCLI)
189 3053 : .SetAvailableInPipelineStep(false);
190 : auto &datasetArg =
191 : AddInputDatasetArg(
192 : &m_inputDataset, GDAL_OF_VECTOR, false,
193 9027 : m_constructorOptions.inputDatasetHelpMsg.empty() &&
194 2921 : m_constructorOptions.inputDatasetMaxCount == 1
195 : ? "Input vector dataset"
196 5974 : : m_constructorOptions.inputDatasetHelpMsg.c_str())
197 3053 : .SetMinCount(m_constructorOptions.inputDatasetRequired ? 1 : 0)
198 3053 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
199 3053 : .SetDatasetInputFlags(m_constructorOptions.inputDatasetInputFlags)
200 3053 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
201 3053 : .SetHiddenForCLI(hiddenForCLI)
202 3053 : .SetAvailableInPipelineStep(false);
203 3053 : if (!m_constructorOptions.inputDatasetAlias.empty())
204 192 : datasetArg.AddAlias(m_constructorOptions.inputDatasetAlias);
205 3053 : if (!m_constructorOptions.inputDatasetMetaVar.empty())
206 3053 : datasetArg.SetMetaVar(m_constructorOptions.inputDatasetMetaVar);
207 3053 : if (m_constructorOptions.inputDatasetPositional && !hiddenForCLI)
208 2735 : datasetArg.SetPositional();
209 3053 : if (m_constructorOptions.inputDatasetRequired && !hiddenForCLI)
210 2735 : datasetArg.SetRequired();
211 3053 : if (m_constructorOptions.addInputLayerNameArgument)
212 : {
213 : auto &layerArg = AddArg(GDAL_ARG_NAME_INPUT_LAYER, 'l',
214 5392 : _("Input layer name(s)"), &m_inputLayerNames)
215 5392 : .AddAlias("layer")
216 2696 : .SetHiddenForCLI(hiddenForCLI)
217 2696 : .SetAvailableInPipelineStep(false);
218 2696 : SetAutoCompleteFunctionForLayerName(layerArg, datasetArg);
219 : }
220 3053 : }
221 :
222 : /************************************************************************/
223 : /* GDALPipelineStepAlgorithm::AddVectorOutputArgs() */
224 : /************************************************************************/
225 :
226 3495 : void GDALPipelineStepAlgorithm::AddVectorOutputArgs(
227 : bool hiddenForCLI, bool shortNameOutputLayerAllowed)
228 : {
229 : AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
230 3495 : /* bGDALGAllowed = */ true)
231 : .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,
232 13980 : {GDAL_DCAP_VECTOR, GDAL_DCAP_CREATE})
233 3495 : .SetHiddenForCLI(hiddenForCLI)
234 3495 : .SetAvailableInPipelineStep(false);
235 3495 : AddOutputOpenOptionsArg(&m_outputOpenOptions)
236 3495 : .SetHiddenForCLI(hiddenForCLI)
237 3495 : .SetAvailableInPipelineStep(false);
238 : auto &outputDatasetArg =
239 : AddOutputDatasetArg(&m_outputDataset, GDAL_OF_VECTOR,
240 3495 : /* positionalAndRequired = */ false)
241 3495 : .SetHiddenForCLI(hiddenForCLI)
242 3495 : .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT)
243 3495 : .SetAvailableInPipelineStep(false);
244 3495 : if (!hiddenForCLI)
245 3309 : outputDatasetArg.SetPositional();
246 3495 : if (!hiddenForCLI && m_constructorOptions.outputDatasetRequired)
247 3249 : outputDatasetArg.SetRequired();
248 :
249 3495 : AddCreationOptionsArg(&m_creationOptions)
250 3495 : .SetHiddenForCLI(hiddenForCLI)
251 3495 : .SetAvailableInPipelineStep(false);
252 3495 : AddLayerCreationOptionsArg(&m_layerCreationOptions)
253 3495 : .SetHiddenForCLI(hiddenForCLI)
254 3495 : .SetAvailableInPipelineStep(false);
255 3495 : AddOverwriteArg(&m_overwrite)
256 3495 : .SetHiddenForCLI(hiddenForCLI)
257 3495 : .SetAvailableInPipelineStep(false);
258 3495 : GDALInConstructionAlgorithmArg *updateArg = nullptr;
259 3495 : if (m_constructorOptions.addUpdateArgument)
260 : {
261 3432 : updateArg = &AddUpdateArg(&m_update)
262 3432 : .SetHiddenForCLI(hiddenForCLI)
263 3432 : .SetAvailableInPipelineStep(false);
264 : }
265 3495 : if (m_constructorOptions.addOverwriteLayerArgument)
266 : {
267 3432 : AddOverwriteLayerArg(&m_overwriteLayer)
268 3432 : .SetHiddenForCLI(hiddenForCLI)
269 3432 : .SetAvailableInPipelineStep(false);
270 : }
271 3495 : constexpr const char *MUTUAL_EXCLUSION_GROUP_APPEND_UPSERT =
272 : "append-upsert";
273 3495 : if (m_constructorOptions.addAppendLayerArgument)
274 : {
275 3300 : AddAppendLayerArg(&m_appendLayer)
276 3300 : .SetHiddenForCLI(hiddenForCLI)
277 6600 : .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_APPEND_UPSERT)
278 3300 : .SetAvailableInPipelineStep(false);
279 : }
280 3495 : if (m_constructorOptions.addUpsertArgument)
281 : {
282 6170 : AddArg("upsert", 0, _("Upsert features (implies 'append')"), &m_upsert)
283 3085 : .SetHiddenForCLI(hiddenForCLI)
284 6170 : .SetMutualExclusionGroup(MUTUAL_EXCLUSION_GROUP_APPEND_UPSERT)
285 :
286 3085 : .SetAvailableInPipelineStep(false)
287 : .AddAction(
288 12 : [updateArg, this]()
289 : {
290 4 : if (m_upsert && updateArg)
291 4 : updateArg->Set(true);
292 6170 : })
293 3085 : .SetCategory(GAAC_ADVANCED);
294 : }
295 3495 : if (m_constructorOptions.addOutputLayerNameArgument)
296 : {
297 3363 : AddOutputLayerNameArg(hiddenForCLI, shortNameOutputLayerAllowed);
298 : }
299 3495 : if (m_constructorOptions.addSkipErrorsArgument)
300 : {
301 : AddArg("skip-errors", 0, _("Skip errors when writing features"),
302 6170 : &m_skipErrors)
303 6170 : .AddHiddenAlias("skip-failures") // For ogr2ogr nostalgic people
304 3085 : .SetAvailableInPipelineStep(false);
305 : }
306 3495 : if (m_constructorOptions.addNoCreateEmptyLayersArgument)
307 : {
308 : AddArg("no-create-empty-layers", 0,
309 : _("Avoid creating layers to which no features will be written"),
310 2360 : &m_noCreateEmptyLayers)
311 1180 : .SetAvailableInPipelineStep(false);
312 : }
313 3495 : }
314 :
315 : /************************************************************************/
316 : /* GDALPipelineStepAlgorithm::AddOutputLayerNameArg() */
317 : /************************************************************************/
318 :
319 3606 : void GDALPipelineStepAlgorithm::AddOutputLayerNameArg(
320 : bool hiddenForCLI, bool shortNameOutputLayerAllowed)
321 : {
322 : AddArg(GDAL_ARG_NAME_OUTPUT_LAYER, shortNameOutputLayerAllowed ? 'l' : 0,
323 : _("Output layer name"),
324 7212 : &m_outputLayerName)
325 7212 : .AddHiddenAlias("nln") // For ogr2ogr nostalgic people
326 3606 : .SetHiddenForCLI(hiddenForCLI)
327 : .SetAvailableInPipelineStep(
328 3606 : m_constructorOptions.outputLayerNameAvailableInPipelineStep);
329 3606 : }
330 :
331 : /************************************************************************/
332 : /* GDALPipelineStepAlgorithm::AddMdimHiddenInputDatasetArg() */
333 : /************************************************************************/
334 :
335 26 : void GDALPipelineStepAlgorithm::AddMdimHiddenInputDatasetArg()
336 : {
337 26 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_MULTIDIM_RASTER, false)
338 26 : .SetMinCount(0)
339 26 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
340 26 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
341 26 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
342 26 : .SetHidden();
343 26 : }
344 :
345 : /************************************************************************/
346 : /* GDALPipelineStepAlgorithm::AddMdimInputArgs() */
347 : /************************************************************************/
348 :
349 382 : void GDALPipelineStepAlgorithm::AddMdimInputArgs(bool openForMixedMdimVector,
350 : bool hiddenForCLI,
351 : bool acceptRaster)
352 : {
353 382 : AddInputFormatsArg(&m_inputFormats)
354 : .AddMetadataItem(
355 : GAAMDI_REQUIRED_CAPABILITIES,
356 : openForMixedMdimVector
357 1146 : ? std::vector<
358 : std::string>{acceptRaster
359 : ? GDAL_ALG_DCAP_RASTER_OR_MULTIDIM_RASTER
360 : : GDAL_DCAP_MULTIDIM_RASTER,
361 0 : GDAL_DCAP_VECTOR}
362 : : std::vector<
363 : std::string>{acceptRaster
364 : ? GDAL_ALG_DCAP_RASTER_OR_MULTIDIM_RASTER
365 1146 : : GDAL_DCAP_MULTIDIM_RASTER})
366 382 : .SetHiddenForCLI(hiddenForCLI)
367 382 : .SetAvailableInPipelineStep(false);
368 382 : AddOpenOptionsArg(&m_openOptions)
369 382 : .SetHiddenForCLI(hiddenForCLI)
370 382 : .SetAvailableInPipelineStep(false);
371 : auto &arg =
372 : AddInputDatasetArg(&m_inputDataset,
373 382 : (acceptRaster ? GDAL_OF_RASTER : 0) |
374 : (openForMixedMdimVector
375 382 : ? (GDAL_OF_MULTIDIM_RASTER | GDAL_OF_VECTOR)
376 : : GDAL_OF_MULTIDIM_RASTER),
377 : false,
378 764 : m_constructorOptions.inputDatasetHelpMsg.c_str())
379 382 : .SetDatasetInputFlags(m_constructorOptions.inputDatasetInputFlags)
380 382 : .SetMinCount(m_constructorOptions.inputDatasetRequired ? 1 : 0)
381 382 : .SetMaxCount(m_constructorOptions.inputDatasetMaxCount)
382 382 : .SetAutoOpenDataset(m_constructorOptions.autoOpenInputDatasets)
383 382 : .SetMetaVar(m_constructorOptions.inputDatasetMetaVar)
384 382 : .SetHiddenForCLI(hiddenForCLI)
385 382 : .SetAvailableInPipelineStep(false);
386 382 : if (m_constructorOptions.inputDatasetPositional && !hiddenForCLI)
387 327 : arg.SetPositional();
388 382 : if (m_constructorOptions.inputDatasetRequired && !hiddenForCLI)
389 327 : arg.SetRequired();
390 382 : if (!m_constructorOptions.inputDatasetAlias.empty())
391 57 : arg.AddAlias(m_constructorOptions.inputDatasetAlias);
392 382 : }
393 :
394 : /************************************************************************/
395 : /* GDALPipelineStepAlgorithm::AddMdimOutputArgs() */
396 : /************************************************************************/
397 :
398 250 : void GDALPipelineStepAlgorithm::AddMdimOutputArgs(bool hiddenForCLI)
399 : {
400 : // Same requirements as the ones checked by GDALMultiDimTranslate(), which
401 : // writes to a classic raster driver when the driver has no
402 : // multidimensional creation capability.
403 : const std::vector<std::string> requiredCapabilities =
404 250 : m_constructorOptions.mdimOutputAcceptsClassicRaster
405 : ? std::vector<
406 : std::string>{GDAL_ALG_DCAP_RASTER_OR_MULTIDIM_RASTER,
407 : GDAL_DCAP_CREATE
408 : "|" GDAL_DCAP_CREATECOPY
409 : "|" GDAL_DCAP_CREATE_MULTIDIMENSIONAL
410 252 : "|" GDAL_DCAP_CREATECOPY_MULTIDIMENSIONAL}
411 1374 : : std::vector<std::string>{GDAL_DCAP_CREATE_MULTIDIMENSIONAL};
412 :
413 250 : m_outputFormatArg =
414 : &(AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
415 250 : /* bGDALGAllowed = */ true)
416 : .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,
417 500 : requiredCapabilities)
418 250 : .SetHiddenForCLI(hiddenForCLI))
419 250 : .SetAvailableInPipelineStep(false);
420 : AddOutputDatasetArg(&m_outputDataset, GDAL_OF_MULTIDIM_RASTER,
421 250 : /* positionalAndRequired = */ !hiddenForCLI,
422 250 : m_constructorOptions.outputDatasetHelpMsg.c_str())
423 250 : .SetHiddenForCLI(hiddenForCLI)
424 250 : .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT)
425 250 : .SetAvailableInPipelineStep(false);
426 250 : AddCreationOptionsArg(&m_creationOptions)
427 250 : .SetHiddenForCLI(hiddenForCLI)
428 250 : .SetAvailableInPipelineStep(false);
429 250 : AddOverwriteArg(&m_overwrite)
430 250 : .SetHiddenForCLI(hiddenForCLI)
431 250 : .SetAvailableInPipelineStep(false);
432 250 : }
433 :
434 : /************************************************************************/
435 : /* GDALPipelineStepAlgorithm::RunImpl() */
436 : /************************************************************************/
437 :
438 3113 : bool GDALPipelineStepAlgorithm::RunImpl(GDALProgressFunc pfnProgress,
439 : void *pProgressData)
440 : {
441 3113 : if (m_standaloneStep)
442 : {
443 1296 : std::unique_ptr<GDALPipelineStepAlgorithm> readAlg;
444 1296 : if (GetInputType() == GDAL_OF_RASTER)
445 800 : readAlg = std::make_unique<GDALRasterReadAlgorithm>();
446 496 : else if (GetInputType() == GDAL_OF_MULTIDIM_RASTER)
447 22 : readAlg = std::make_unique<GDALMdimReadAlgorithm>();
448 : else
449 474 : readAlg = std::make_unique<GDALVectorReadAlgorithm>();
450 10842 : for (auto &arg : readAlg->GetArgs())
451 : {
452 9546 : auto stepArg = GetArg(arg->GetName());
453 10859 : if (stepArg && stepArg->IsExplicitlySet() &&
454 1313 : stepArg->GetType() == arg->GetType())
455 : {
456 1310 : arg->SetSkipIfAlreadySet(true);
457 1310 : arg->SetFrom(*stepArg);
458 : }
459 : }
460 :
461 0 : std::unique_ptr<GDALPipelineStepAlgorithm> writeAlg;
462 1296 : if (GetOutputType() == GDAL_OF_RASTER)
463 : {
464 571 : if (GetName() == GDALRasterInfoAlgorithm::NAME)
465 26 : writeAlg = std::make_unique<GDALRasterInfoAlgorithm>();
466 545 : else if (GetName() == GDALRasterCompareAlgorithm::NAME)
467 0 : writeAlg = std::make_unique<GDALRasterCompareAlgorithm>();
468 : else
469 545 : writeAlg = std::make_unique<GDALRasterWriteAlgorithm>();
470 : }
471 725 : else if (GetOutputType() == GDAL_OF_MULTIDIM_RASTER)
472 : {
473 13 : if (GetName() == GDALMdimInfoAlgorithm::NAME)
474 10 : writeAlg = std::make_unique<GDALMdimInfoAlgorithm>();
475 : else
476 3 : writeAlg = std::make_unique<GDALMdimWriteAlgorithm>();
477 : }
478 : else
479 : {
480 712 : if (GetName() == GDALVectorInfoAlgorithm::NAME)
481 27 : writeAlg = std::make_unique<GDALVectorInfoAlgorithm>();
482 : else
483 685 : writeAlg = std::make_unique<GDALVectorWriteAlgorithm>();
484 : }
485 21039 : for (auto &arg : writeAlg->GetArgs())
486 : {
487 19743 : auto stepArg = GetArg(arg->GetName());
488 23027 : if (stepArg && stepArg->IsExplicitlySet() &&
489 3284 : stepArg->GetType() == arg->GetType())
490 : {
491 3278 : arg->SetSkipIfAlreadySet(true);
492 3278 : arg->SetFrom(*stepArg);
493 : }
494 : }
495 :
496 1296 : const bool bIsStreaming = m_format == "stream";
497 :
498 : // Already checked by GDALAlgorithm::Run()
499 1296 : CPLAssert(!m_executionForStreamOutput || bIsStreaming);
500 :
501 1296 : bool ret = false;
502 1352 : if (!m_outputVRTCompatible &&
503 112 : (EQUAL(m_format.c_str(), "VRT") ||
504 56 : (m_format.empty() &&
505 1297 : EQUAL(CPLGetExtensionSafe(m_outputDataset.GetName().c_str())
506 : .c_str(),
507 : "VRT"))))
508 : {
509 0 : ReportError(CE_Failure, CPLE_NotSupported,
510 : "VRT output is not supported. Consider using the "
511 : "GDALG driver instead (files with .gdalg.json "
512 : "extension)");
513 : }
514 1296 : else if (readAlg->Run())
515 : {
516 1292 : auto outputArg = GetArg(GDAL_ARG_NAME_OUTPUT);
517 : const bool bOutputSpecified =
518 2297 : outputArg && outputArg->IsExplicitlySet() &&
519 1005 : (outputArg->GetType() == GAAT_DATASET ||
520 3 : outputArg->GetType() == GAAT_DATASET_LIST);
521 :
522 1292 : m_inputDataset.clear();
523 1292 : m_inputDataset.resize(1);
524 1292 : m_inputDataset[0].Set(readAlg->m_outputDataset.GetDatasetRef());
525 1292 : if (bOutputSpecified)
526 999 : m_outputDataset.Set(nullptr);
527 :
528 : std::unique_ptr<void, decltype(&GDALDestroyScaledProgress)>
529 2584 : pScaledData(nullptr, GDALDestroyScaledProgress);
530 :
531 : const bool bCanHandleNextStep =
532 1292 : !bIsStreaming && CanHandleNextStep(writeAlg.get());
533 :
534 1292 : GDALPipelineStepRunContext stepCtxt;
535 1292 : if (pfnProgress && GetName() == GDALRasterCompareAlgorithm::NAME)
536 : {
537 11 : stepCtxt.m_pfnProgress = pfnProgress;
538 11 : stepCtxt.m_pProgressData = pProgressData;
539 : }
540 1335 : else if (pfnProgress &&
541 54 : (bCanHandleNextStep || !IsNativelyStreamingCompatible()))
542 : {
543 52 : pScaledData.reset(GDALCreateScaledProgress(
544 22 : 0.0, bIsStreaming || bCanHandleNextStep ? 1.0 : 0.5,
545 : pfnProgress, pProgressData));
546 30 : stepCtxt.m_pfnProgress =
547 30 : pScaledData ? GDALScaledProgress : nullptr;
548 30 : stepCtxt.m_pProgressData = pScaledData.get();
549 : }
550 :
551 1292 : if (bCanHandleNextStep)
552 52 : stepCtxt.m_poNextUsableStep = writeAlg.get();
553 1292 : if (RunPreStepPipelineValidations() && RunStep(stepCtxt))
554 : {
555 1178 : if (bCanHandleNextStep || !bOutputSpecified)
556 : {
557 331 : ret = true;
558 : }
559 : else
560 : {
561 847 : writeAlg->m_outputVRTCompatible = m_outputVRTCompatible;
562 847 : writeAlg->m_quiet = m_quiet;
563 :
564 1694 : std::vector<GDALArgDatasetValue> inputDataset(1);
565 847 : inputDataset[0].Set(m_outputDataset.GetDatasetRef());
566 847 : auto inputArg = writeAlg->GetArg(GDAL_ARG_NAME_INPUT);
567 847 : CPLAssert(inputArg);
568 847 : inputArg->Set(std::move(inputDataset));
569 :
570 847 : if (pfnProgress)
571 : {
572 41 : pScaledData.reset(GDALCreateScaledProgress(
573 41 : IsNativelyStreamingCompatible() ? 0.0 : 0.5, 1.0,
574 : pfnProgress, pProgressData));
575 : }
576 847 : stepCtxt.m_pfnProgress =
577 847 : pScaledData ? GDALScaledProgress : nullptr;
578 847 : stepCtxt.m_pProgressData = pScaledData.get();
579 1694 : if (writeAlg->ValidateArguments() &&
580 847 : writeAlg->RunStep(stepCtxt))
581 : {
582 820 : if (pfnProgress)
583 40 : pfnProgress(1.0, "", pProgressData);
584 :
585 820 : m_outputDataset.Set(
586 820 : writeAlg->m_outputDataset.GetDatasetRef());
587 820 : ret = true;
588 : }
589 : }
590 : }
591 : }
592 :
593 1296 : return ret;
594 : }
595 : else
596 : {
597 1817 : GDALPipelineStepRunContext stepCtxt;
598 1817 : stepCtxt.m_pfnProgress = pfnProgress;
599 1817 : stepCtxt.m_pProgressData = pProgressData;
600 1817 : return RunPreStepPipelineValidations() && RunStep(stepCtxt);
601 : }
602 : }
603 :
604 : /************************************************************************/
605 : /* SetInputDataset() */
606 : /************************************************************************/
607 :
608 27 : void GDALPipelineStepAlgorithm::SetInputDataset(GDALDataset *poDS)
609 : {
610 27 : auto arg = GetArg(GDAL_ARG_NAME_INPUT);
611 27 : if (arg)
612 : {
613 27 : auto &val = arg->Get<std::vector<GDALArgDatasetValue>>();
614 27 : val.resize(1);
615 27 : val[0].Set(poDS);
616 27 : arg->NotifyValueSet();
617 27 : arg->SetSkipIfAlreadySet();
618 : }
619 27 : }
620 :
621 : /************************************************************************/
622 : /* ProcessGDALGOutput() */
623 : /************************************************************************/
624 :
625 : GDALAlgorithm::ProcessGDALGOutputRet
626 4297 : GDALPipelineStepAlgorithm::ProcessGDALGOutput()
627 : {
628 4297 : if (m_standaloneStep)
629 : {
630 2397 : return GDALAlgorithm::ProcessGDALGOutput();
631 : }
632 : else
633 : {
634 : // GDALAbstractPipelineAlgorithm<StepAlgorithm>::RunStep() might
635 : // actually detect a GDALG output request and process it.
636 1900 : return GDALAlgorithm::ProcessGDALGOutputRet::NOT_GDALG;
637 : }
638 : }
639 :
640 : /************************************************************************/
641 : /* GDALPipelineStepAlgorithm::CheckSafeForStreamOutput() */
642 : /************************************************************************/
643 :
644 98 : bool GDALPipelineStepAlgorithm::CheckSafeForStreamOutput()
645 : {
646 98 : if (m_standaloneStep)
647 : {
648 50 : return GDALAlgorithm::CheckSafeForStreamOutput();
649 : }
650 : else
651 : {
652 : // The check is actually done in
653 : // GDALAbstractPipelineAlgorithm<StepAlgorithm>::RunStep()
654 : // so return true for now.
655 48 : return true;
656 : }
657 : }
658 :
659 : /************************************************************************/
660 : /* GDALPipelineStepAlgorithm::Finalize() */
661 : /************************************************************************/
662 :
663 1620 : bool GDALPipelineStepAlgorithm::Finalize()
664 : {
665 1620 : bool ret = GDALAlgorithm::Finalize();
666 3019 : for (auto &argValue : m_inputDataset)
667 1399 : ret = argValue.Close() && ret;
668 1620 : ret = m_outputDataset.Close() && ret;
669 1620 : return ret;
670 : }
671 :
672 : /************************************************************************/
673 : /* CreateDatasetSingleOutputLayerIfNeeded() */
674 : /************************************************************************/
675 :
676 68 : bool GDALPipelineStepAlgorithm::CreateDatasetSingleOutputLayerIfNeeded(
677 : GDALPipelineStepRunContext &ctxt, const std::string &defaultLayerName,
678 : GDALDataset *&poDstDS, bool &bTemporaryFile,
679 : std::unique_ptr<GDALDataset> &poNewRetDS, std::string &outputLayerName,
680 : OGRLayer *&poDstLayer)
681 : {
682 68 : auto poWriteStep = ctxt.m_poNextUsableStep ? ctxt.m_poNextUsableStep : this;
683 136 : std::string outputFilename = poWriteStep->GetOutputDataset().GetName();
684 :
685 68 : poDstDS = poWriteStep->GetOutputDataset().GetDatasetRef();
686 68 : poNewRetDS.reset();
687 68 : bTemporaryFile = false;
688 68 : poDstLayer = nullptr;
689 68 : GDALDriver *poDstDriver = nullptr;
690 68 : if (!poDstDS)
691 : {
692 48 : auto poDriverManager = GetGDALDriverManager();
693 48 : std::string format = poWriteStep->GetOutputFormat();
694 48 : if (m_standaloneStep || (ctxt.m_poNextUsableStep && format.empty()))
695 : {
696 41 : if (format.empty())
697 : {
698 : const auto aosFormats =
699 : CPLStringList(GDALGetOutputDriversForDatasetName(
700 : outputFilename.c_str(), GDAL_OF_VECTOR,
701 : /* bSingleMatch = */ true,
702 25 : /* bWarn = */ true));
703 25 : if (aosFormats.size() != 1)
704 : {
705 2 : ReportError(CE_Failure, CPLE_AppDefined,
706 : "Cannot guess driver for %s",
707 : outputFilename.c_str());
708 2 : return false;
709 : }
710 23 : format = aosFormats[0];
711 : }
712 : }
713 7 : else if (!ctxt.m_poNextUsableStep)
714 : {
715 5 : poDstDriver = poDriverManager->GetDriverByName("GPKG");
716 5 : if (poDstDriver)
717 : {
718 5 : bTemporaryFile = true;
719 : outputFilename =
720 5 : CPLGenerateTempFilenameSafe(
721 15 : std::string("_").append(defaultLayerName).c_str()) +
722 5 : ".gpkg";
723 5 : format = "GPKG";
724 : }
725 : else
726 0 : format = "MEM";
727 : }
728 :
729 46 : if (!poDstDriver)
730 41 : poDstDriver = poDriverManager->GetDriverByName(format.c_str());
731 46 : if (!poDstDriver)
732 : {
733 0 : ReportError(CE_Failure, CPLE_AppDefined, "Cannot find driver %s",
734 : format.c_str());
735 0 : return false;
736 : }
737 :
738 46 : poNewRetDS.reset(poDstDriver->Create(
739 : outputFilename.c_str(), 0, 0, 0, GDT_Unknown,
740 92 : CPLStringList(poWriteStep->GetCreationOptions()).List()));
741 46 : if (!poNewRetDS)
742 2 : return false;
743 :
744 44 : if (bTemporaryFile)
745 5 : poNewRetDS->MarkSuppressOnClose();
746 :
747 44 : poDstDS = poNewRetDS.get();
748 : }
749 : else
750 : {
751 20 : poDstDriver = poDstDS->GetDriver();
752 : }
753 :
754 64 : outputLayerName = poWriteStep->GetOutputLayerName();
755 64 : if (outputLayerName.empty() && poDstDS->GetLayerCount() > 1)
756 : {
757 3 : ReportError(CE_Failure, CPLE_AppDefined, "--%s must be specified",
758 : GDAL_ARG_NAME_OUTPUT_LAYER);
759 3 : return false;
760 : }
761 :
762 61 : if (poDstDriver && EQUAL(poDstDriver->GetDescription(), "ESRI Shapefile") &&
763 81 : (EQUAL(CPLGetExtensionSafe(poDstDS->GetDescription()).c_str(), "shp") ||
764 64 : EQUAL(CPLGetExtensionSafe(poDstDS->GetDescription()).c_str(),
765 122 : "shz")) &&
766 20 : poDstDS->GetLayerCount() <= 1)
767 : {
768 20 : outputLayerName = CPLGetBasenameSafe(poDstDS->GetDescription());
769 : }
770 61 : if (outputLayerName.empty())
771 33 : outputLayerName = defaultLayerName;
772 :
773 61 : poDstLayer = poDstDS->GetLayerByName(outputLayerName.c_str());
774 61 : if (poDstLayer)
775 : {
776 13 : if (poWriteStep->GetOverwriteLayer())
777 : {
778 5 : int iLayer = -1;
779 5 : const int nLayerCount = poDstDS->GetLayerCount();
780 6 : for (iLayer = 0; iLayer < nLayerCount; iLayer++)
781 : {
782 6 : if (poDstDS->GetLayer(iLayer) == poDstLayer)
783 5 : break;
784 : }
785 :
786 5 : if (iLayer < nLayerCount)
787 : {
788 5 : if (poDstDS->DeleteLayer(iLayer) != OGRERR_NONE)
789 : {
790 2 : ReportError(CE_Failure, CPLE_AppDefined,
791 : "Cannot delete layer '%s'",
792 : outputLayerName.c_str());
793 2 : return false;
794 : }
795 : }
796 3 : poDstLayer = nullptr;
797 : }
798 8 : else if (!poWriteStep->GetAppendLayer())
799 : {
800 3 : ReportError(CE_Failure, CPLE_AppDefined,
801 : "Layer '%s' already exists. Specify the "
802 : "--%s option to overwrite it, or --%s "
803 : "to append to it.",
804 : outputLayerName.c_str(), GDAL_ARG_NAME_OVERWRITE_LAYER,
805 : GDAL_ARG_NAME_APPEND);
806 3 : return false;
807 : }
808 : }
809 48 : else if (poWriteStep->GetAppendLayer() || poWriteStep->GetOverwriteLayer())
810 : {
811 3 : ReportError(CE_Failure, CPLE_AppDefined, "Cannot find layer '%s'",
812 : outputLayerName.c_str());
813 3 : return false;
814 : }
815 :
816 53 : return true;
817 : }
818 :
819 : GDALAlgorithmStepRegistry::~GDALAlgorithmStepRegistry() = default;
820 :
821 : /************************************************************************/
822 : /* GDALPipelineAlgorithm */
823 : /************************************************************************/
824 :
825 338 : GDALPipelineAlgorithm::GDALPipelineAlgorithm()
826 : : GDALAbstractPipelineAlgorithm(
827 : NAME, DESCRIPTION, HELP_URL,
828 338 : ConstructorOptions().SetStandaloneStep(false))
829 : {
830 338 : m_supportsStreamedOutput = true;
831 :
832 338 : AddProgressArg();
833 : AddInputDatasetArg(&m_inputDataset, GDAL_OF_RASTER | GDAL_OF_VECTOR,
834 338 : /* positionalAndRequired = */ false)
835 338 : .SetMinCount(1)
836 338 : .SetMaxCount(INT_MAX)
837 338 : .SetHiddenForCLI();
838 : AddOutputDatasetArg(&m_outputDataset, GDAL_OF_RASTER | GDAL_OF_VECTOR,
839 338 : /* positionalAndRequired = */ false)
840 338 : .SetHiddenForCLI()
841 338 : .SetDatasetInputFlags(GADV_NAME | GADV_OBJECT);
842 : AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
843 338 : /* bGDALGAllowed = */ true)
844 338 : .SetHiddenForCLI();
845 676 : AddArg("pipeline", 0, _("Pipeline string or filename"), &m_pipeline)
846 338 : .SetHiddenForCLI()
847 338 : .SetPositional();
848 :
849 338 : AddOutputStringArg(&m_output).SetHiddenForCLI();
850 338 : AddStdoutArg(&m_stdout);
851 :
852 338 : AllowArbitraryLongNameArgs();
853 :
854 338 : GDALRasterPipelineAlgorithm::RegisterAlgorithms(m_stepRegistry, true);
855 338 : GDALVectorPipelineAlgorithm::RegisterAlgorithms(m_stepRegistry, true);
856 338 : m_stepRegistry.Register<GDALExternalRasterOrVectorAlgorithm>();
857 338 : m_stepRegistry.Register<GDALRasterAsFeaturesAlgorithm>();
858 338 : m_stepRegistry.Register<GDALRasterContourAlgorithm>();
859 338 : m_stepRegistry.Register<GDALRasterFootprintAlgorithm>();
860 338 : m_stepRegistry.Register<GDALRasterPixelInfoAlgorithm>();
861 338 : m_stepRegistry.Register<GDALRasterPolygonizeAlgorithm>();
862 338 : m_stepRegistry.Register<GDALRasterZonalStatsAlgorithm>();
863 338 : m_stepRegistry.Register<GDALVectorGridAlgorithm>();
864 338 : m_stepRegistry.Register<GDALVectorRasterizeAlgorithm>();
865 338 : }
866 :
867 : /************************************************************************/
868 : /* GDALPipelineAlgorithm::GetUsageForCLI() */
869 : /************************************************************************/
870 :
871 : std::string
872 19 : GDALPipelineAlgorithm::GetUsageForCLI(bool shortUsage,
873 : const UsageOptions &usageOptions) const
874 : {
875 19 : UsageOptions stepUsageOptions;
876 19 : stepUsageOptions.isPipelineStep = true;
877 :
878 19 : if (!m_helpDocCategory.empty() && m_helpDocCategory != "main")
879 : {
880 4 : auto alg = GetStepAlg(m_helpDocCategory);
881 2 : if (alg)
882 : {
883 3 : alg->SetCallPath({CPLString(m_helpDocCategory)
884 2 : .replaceAll(RASTER_SUFFIX, "")
885 3 : .replaceAll(VECTOR_SUFFIX, "")});
886 1 : alg->GetArg("help-doc")->Set(true);
887 1 : return alg->GetUsageForCLI(shortUsage, stepUsageOptions);
888 : }
889 : else
890 : {
891 1 : fprintf(stderr, "ERROR: unknown pipeline step '%s'\n",
892 : m_helpDocCategory.c_str());
893 : return CPLSPrintf("ERROR: unknown pipeline step '%s'\n",
894 1 : m_helpDocCategory.c_str());
895 : }
896 : }
897 :
898 17 : UsageOptions usageOptionsMain(usageOptions);
899 17 : usageOptionsMain.isPipelineMain = true;
900 : std::string ret =
901 34 : GDALAlgorithm::GetUsageForCLI(shortUsage, usageOptionsMain);
902 17 : if (shortUsage)
903 15 : return ret;
904 :
905 : ret +=
906 : "\n<PIPELINE> is of the form: read|calc|concat|create|mosaic|stack "
907 : "[READ-OPTIONS] "
908 2 : "( ! <STEP-NAME> [STEP-OPTIONS] )* ! write!info!tile [WRITE-OPTIONS]\n";
909 :
910 2 : if (m_helpDocCategory == "main")
911 : {
912 1 : return ret;
913 : }
914 :
915 1 : ret += '\n';
916 1 : ret += "Example: 'gdal pipeline --progress ! read in.tif ! \\\n";
917 1 : ret += " rasterize --size 256 256 ! buffer 20 ! ";
918 1 : ret += "write out.gpkg --overwrite'\n";
919 1 : ret += '\n';
920 1 : ret += "Potential steps are:\n";
921 :
922 88 : for (const std::string &name : m_stepRegistry.GetNames())
923 : {
924 174 : auto alg = GetStepAlg(name);
925 87 : assert(alg);
926 87 : auto [options, maxOptLen] = alg->GetArgNamesForCLI();
927 87 : stepUsageOptions.maxOptLen =
928 87 : std::max(stepUsageOptions.maxOptLen, maxOptLen);
929 : }
930 :
931 : {
932 1 : ret += '\n';
933 1 : auto alg = std::make_unique<GDALRasterReadAlgorithm>();
934 2 : alg->SetCallPath({alg->GetName()});
935 1 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
936 : }
937 : {
938 1 : ret += '\n';
939 1 : auto alg = std::make_unique<GDALVectorReadAlgorithm>();
940 2 : alg->SetCallPath({alg->GetName()});
941 1 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
942 : }
943 88 : for (const std::string &name : m_stepRegistry.GetNames())
944 : {
945 174 : auto alg = GetStepAlg(name);
946 87 : assert(alg);
947 96 : if (alg->CanBeFirstStep() && !alg->CanBeMiddleStep() &&
948 101 : !alg->IsHidden() &&
949 5 : !STARTS_WITH(name.c_str(), GDALRasterReadAlgorithm::NAME))
950 : {
951 3 : ret += '\n';
952 6 : alg->SetCallPath({name});
953 3 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
954 : }
955 : }
956 88 : for (const std::string &name : m_stepRegistry.GetNames())
957 : {
958 174 : auto alg = GetStepAlg(name);
959 87 : assert(alg);
960 87 : if (alg->CanBeMiddleStep() && !alg->IsHidden())
961 : {
962 74 : ret += '\n';
963 222 : alg->SetCallPath({CPLString(alg->GetName())
964 148 : .replaceAll(RASTER_SUFFIX, "")
965 222 : .replaceAll(VECTOR_SUFFIX, "")});
966 74 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
967 : }
968 : }
969 88 : for (const std::string &name : m_stepRegistry.GetNames())
970 : {
971 174 : auto alg = GetStepAlg(name);
972 87 : assert(alg);
973 100 : if (alg->CanBeLastStep() && !alg->CanBeMiddleStep() &&
974 108 : !alg->IsHidden() &&
975 8 : !STARTS_WITH(name.c_str(), GDALRasterWriteAlgorithm::NAME))
976 : {
977 6 : ret += '\n';
978 12 : alg->SetCallPath({name});
979 6 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
980 : }
981 : }
982 : {
983 1 : ret += '\n';
984 1 : auto alg = std::make_unique<GDALRasterWriteAlgorithm>();
985 2 : alg->SetCallPath({alg->GetName()});
986 1 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
987 : }
988 : {
989 1 : ret += '\n';
990 1 : auto alg = std::make_unique<GDALVectorWriteAlgorithm>();
991 2 : alg->SetCallPath({alg->GetName()});
992 1 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
993 : }
994 :
995 1 : ret += GetUsageForCLIEnd();
996 :
997 1 : return ret;
998 : }
999 :
1000 : //! @endcond
|
