Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: GDAL
4 : * Purpose: gdal "raster pipeline" subcommand
5 : * Author: Even Rouault <even dot rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2024, Even Rouault <even dot rouault at spatialys.com>
9 : *
10 : * SPDX-License-Identifier: MIT
11 : ****************************************************************************/
12 :
13 : #include "gdalalg_raster_pipeline.h"
14 : #include "gdalalg_raster_read.h"
15 : #include "gdalalg_raster_calc.h"
16 : #include "gdalalg_raster_aspect.h"
17 : #include "gdalalg_raster_clip.h"
18 : #include "gdalalg_raster_color_map.h"
19 : #include "gdalalg_raster_color_merge.h"
20 : #include "gdalalg_raster_edit.h"
21 : #include "gdalalg_raster_fill_nodata.h"
22 : #include "gdalalg_raster_hillshade.h"
23 : #include "gdalalg_raster_mosaic.h"
24 : #include "gdalalg_raster_nodata_to_alpha.h"
25 : #include "gdalalg_raster_pansharpen.h"
26 : #include "gdalalg_raster_proximity.h"
27 : #include "gdalalg_raster_reclassify.h"
28 : #include "gdalalg_raster_reproject.h"
29 : #include "gdalalg_raster_resize.h"
30 : #include "gdalalg_raster_rgb_to_palette.h"
31 : #include "gdalalg_raster_roughness.h"
32 : #include "gdalalg_raster_scale.h"
33 : #include "gdalalg_raster_select.h"
34 : #include "gdalalg_raster_set_type.h"
35 : #include "gdalalg_raster_sieve.h"
36 : #include "gdalalg_raster_slope.h"
37 : #include "gdalalg_raster_stack.h"
38 : #include "gdalalg_raster_write.h"
39 : #include "gdalalg_raster_tpi.h"
40 : #include "gdalalg_raster_tri.h"
41 : #include "gdalalg_raster_unscale.h"
42 : #include "gdalalg_raster_viewshed.h"
43 :
44 : #include "cpl_conv.h"
45 : #include "cpl_progress.h"
46 : #include "cpl_string.h"
47 : #include "cpl_vsi.h"
48 : #include "gdal_priv.h"
49 : #include "gdal_utils.h"
50 :
51 : #include <algorithm>
52 : #include <array>
53 : #include <cassert>
54 :
55 : //! @cond Doxygen_Suppress
56 :
57 : #ifndef _
58 : #define _(x) (x)
59 : #endif
60 :
61 : /************************************************************************/
62 : /* GDALRasterPipelineStepAlgorithm::GDALRasterPipelineStepAlgorithm() */
63 : /************************************************************************/
64 :
65 1193 : GDALRasterPipelineStepAlgorithm::GDALRasterPipelineStepAlgorithm(
66 : const std::string &name, const std::string &description,
67 1193 : const std::string &helpURL, bool standaloneStep)
68 : : GDALRasterPipelineStepAlgorithm(
69 : name, description, helpURL,
70 1193 : ConstructorOptions().SetStandaloneStep(standaloneStep))
71 : {
72 1193 : }
73 :
74 : /************************************************************************/
75 : /* GDALRasterPipelineStepAlgorithm::GDALRasterPipelineStepAlgorithm() */
76 : /************************************************************************/
77 :
78 1994 : GDALRasterPipelineStepAlgorithm::GDALRasterPipelineStepAlgorithm(
79 : const std::string &name, const std::string &description,
80 1994 : const std::string &helpURL, const ConstructorOptions &options)
81 1994 : : GDALPipelineStepAlgorithm(name, description, helpURL, options)
82 : {
83 1994 : if (m_standaloneStep)
84 : {
85 488 : m_supportsStreamedOutput = true;
86 :
87 488 : if (m_constructorOptions.addDefaultArguments)
88 : {
89 332 : AddRasterInputArgs(false, false);
90 332 : AddProgressArg();
91 332 : AddRasterOutputArgs(false);
92 : }
93 : }
94 1506 : else if (m_constructorOptions.addDefaultArguments)
95 : {
96 861 : AddRasterHiddenInputDatasetArg();
97 : }
98 1994 : }
99 :
100 : GDALRasterPipelineStepAlgorithm::~GDALRasterPipelineStepAlgorithm() = default;
101 :
102 : /************************************************************************/
103 : /* GDALRasterPipelineStepAlgorithm::SetOutputVRTCompatible() */
104 : /************************************************************************/
105 :
106 219 : void GDALRasterPipelineStepAlgorithm::SetOutputVRTCompatible(bool b)
107 : {
108 219 : m_outputVRTCompatible = b;
109 219 : if (m_outputFormatArg)
110 : {
111 91 : m_outputFormatArg->AddMetadataItem(GAAMDI_VRT_COMPATIBLE,
112 182 : {b ? "true" : "false"});
113 : }
114 219 : }
115 :
116 : /************************************************************************/
117 : /* GDALRasterPipelineAlgorithm::GDALRasterPipelineAlgorithm() */
118 : /************************************************************************/
119 :
120 99 : GDALRasterPipelineAlgorithm::GDALRasterPipelineAlgorithm(
121 99 : bool openForMixedRasterVector)
122 : : GDALAbstractPipelineAlgorithm<GDALRasterPipelineStepAlgorithm>(
123 : NAME, DESCRIPTION, HELP_URL,
124 0 : ConstructorOptions()
125 99 : .SetAddDefaultArguments(false)
126 198 : .SetInputDatasetMaxCount(INT_MAX))
127 : {
128 99 : m_supportsStreamedOutput = true;
129 :
130 99 : AddRasterInputArgs(openForMixedRasterVector, /* hiddenForCLI = */ true);
131 99 : AddProgressArg();
132 198 : AddArg("pipeline", 0, _("Pipeline string"), &m_pipeline)
133 99 : .SetHiddenForCLI()
134 99 : .SetPositional();
135 99 : AddRasterOutputArgs(/* hiddenForCLI = */ true);
136 :
137 99 : RegisterAlgorithms(m_stepRegistry, false);
138 99 : }
139 :
140 : /************************************************************************/
141 : /* GDALRasterPipelineAlgorithm::RegisterAlgorithms() */
142 : /************************************************************************/
143 :
144 : /* static */
145 148 : void GDALRasterPipelineAlgorithm::RegisterAlgorithms(
146 : GDALAlgorithmRegistry ®istry, bool forMixedPipeline)
147 : {
148 296 : GDALAlgorithmRegistry::AlgInfo algInfo;
149 :
150 888 : const auto addSuffixIfNeeded = [forMixedPipeline](const char *name)
151 : {
152 1182 : return forMixedPipeline ? std::string(name).append(RASTER_SUFFIX)
153 2070 : : std::string(name);
154 148 : };
155 :
156 148 : algInfo.m_name = addSuffixIfNeeded(GDALRasterReadAlgorithm::NAME);
157 98 : algInfo.m_creationFunc = []() -> std::unique_ptr<GDALAlgorithm>
158 246 : { return std::make_unique<GDALRasterReadAlgorithm>(); };
159 148 : registry.Register(algInfo);
160 :
161 148 : registry.Register<GDALRasterCalcAlgorithm>();
162 :
163 148 : algInfo.m_name = addSuffixIfNeeded(GDALRasterWriteAlgorithm::NAME);
164 130 : algInfo.m_creationFunc = []() -> std::unique_ptr<GDALAlgorithm>
165 278 : { return std::make_unique<GDALRasterWriteAlgorithm>(); };
166 148 : registry.Register(algInfo);
167 :
168 148 : registry.Register<GDALRasterAspectAlgorithm>();
169 :
170 148 : algInfo.m_name = addSuffixIfNeeded(GDALRasterClipAlgorithm::NAME);
171 19 : algInfo.m_creationFunc = []() -> std::unique_ptr<GDALAlgorithm>
172 167 : { return std::make_unique<GDALRasterClipAlgorithm>(); };
173 148 : registry.Register(algInfo);
174 :
175 148 : registry.Register<GDALRasterColorMapAlgorithm>();
176 148 : registry.Register<GDALRasterColorMergeAlgorithm>();
177 :
178 148 : algInfo.m_name = addSuffixIfNeeded(GDALRasterEditAlgorithm::NAME);
179 33 : algInfo.m_creationFunc = []() -> std::unique_ptr<GDALAlgorithm>
180 181 : { return std::make_unique<GDALRasterEditAlgorithm>(); };
181 148 : registry.Register(algInfo);
182 :
183 148 : registry.Register<GDALRasterNoDataToAlphaAlgorithm>();
184 148 : registry.Register<GDALRasterFillNodataAlgorithm>();
185 148 : registry.Register<GDALRasterHillshadeAlgorithm>();
186 148 : registry.Register<GDALRasterMosaicAlgorithm>();
187 148 : registry.Register<GDALRasterPansharpenAlgorithm>();
188 148 : registry.Register<GDALRasterProximityAlgorithm>();
189 148 : registry.Register<GDALRasterReclassifyAlgorithm>();
190 :
191 148 : algInfo.m_name = addSuffixIfNeeded(GDALRasterReprojectAlgorithm::NAME);
192 41 : algInfo.m_creationFunc = []() -> std::unique_ptr<GDALAlgorithm>
193 189 : { return std::make_unique<GDALRasterReprojectAlgorithm>(); };
194 148 : registry.Register(algInfo);
195 :
196 148 : registry.Register<GDALRasterResizeAlgorithm>();
197 148 : registry.Register<GDALRasterRGBToPaletteAlgorithm>();
198 148 : registry.Register<GDALRasterRoughnessAlgorithm>();
199 148 : registry.Register<GDALRasterScaleAlgorithm>();
200 :
201 148 : algInfo.m_name = addSuffixIfNeeded(GDALRasterSelectAlgorithm::NAME);
202 20 : algInfo.m_creationFunc = []() -> std::unique_ptr<GDALAlgorithm>
203 168 : { return std::make_unique<GDALRasterSelectAlgorithm>(); };
204 148 : registry.Register(algInfo);
205 :
206 148 : registry.Register<GDALRasterSetTypeAlgorithm>();
207 148 : registry.Register<GDALRasterSieveAlgorithm>();
208 148 : registry.Register<GDALRasterSlopeAlgorithm>();
209 148 : registry.Register<GDALRasterStackAlgorithm>();
210 148 : registry.Register<GDALRasterTPIAlgorithm>();
211 148 : registry.Register<GDALRasterTRIAlgorithm>();
212 148 : registry.Register<GDALRasterUnscaleAlgorithm>();
213 148 : registry.Register<GDALRasterViewshedAlgorithm>();
214 148 : }
215 :
216 : /************************************************************************/
217 : /* GDALRasterPipelineAlgorithm::ParseCommandLineArguments() */
218 : /************************************************************************/
219 :
220 87 : bool GDALRasterPipelineAlgorithm::ParseCommandLineArguments(
221 : const std::vector<std::string> &args)
222 : {
223 99 : if (args.size() == 1 && (args[0] == "-h" || args[0] == "--help" ||
224 12 : args[0] == "help" || args[0] == "--json-usage"))
225 : {
226 2 : return GDALAlgorithm::ParseCommandLineArguments(args);
227 : }
228 85 : else if (args.size() == 1 && STARTS_WITH(args[0].c_str(), "--help-doc="))
229 : {
230 3 : m_helpDocCategory = args[0].substr(strlen("--help-doc="));
231 6 : return GDALAlgorithm::ParseCommandLineArguments({"--help-doc"});
232 : }
233 :
234 620 : for (const auto &arg : args)
235 : {
236 540 : if (arg.find("--pipeline") == 0)
237 2 : return GDALAlgorithm::ParseCommandLineArguments(args);
238 :
239 : // gdal raster pipeline [--progress] "read in.tif ..."
240 539 : if (arg.find("read ") == 0)
241 1 : return GDALAlgorithm::ParseCommandLineArguments(args);
242 : }
243 :
244 80 : if (!m_steps.empty())
245 : {
246 1 : ReportError(CE_Failure, CPLE_AppDefined,
247 : "ParseCommandLineArguments() can only be called once per "
248 : "instance.");
249 1 : return false;
250 : }
251 :
252 : struct Step
253 : {
254 : std::unique_ptr<GDALRasterPipelineStepAlgorithm> alg{};
255 : std::vector<std::string> args{};
256 : };
257 :
258 158 : std::vector<Step> steps;
259 79 : steps.resize(1);
260 :
261 601 : for (const auto &arg : args)
262 : {
263 527 : if (arg == "--progress")
264 : {
265 2 : m_progressBarRequested = true;
266 2 : continue;
267 : }
268 :
269 525 : if (IsCalledFromCommandLine() && (arg == "-h" || arg == "--help"))
270 : {
271 3 : if (!steps.back().alg)
272 1 : steps.pop_back();
273 3 : if (steps.empty())
274 : {
275 5 : return GDALAlgorithm::ParseCommandLineArguments(args);
276 : }
277 : else
278 : {
279 2 : m_stepOnWhichHelpIsRequested = std::move(steps.back().alg);
280 2 : return true;
281 : }
282 : }
283 :
284 522 : auto &curStep = steps.back();
285 :
286 522 : if (arg == "!" || arg == "|")
287 : {
288 119 : if (curStep.alg)
289 : {
290 109 : steps.resize(steps.size() + 1);
291 : }
292 : }
293 : #ifdef GDAL_PIPELINE_PROJ_NOSTALGIA
294 403 : else if (arg == "+step")
295 : {
296 4 : if (curStep.alg)
297 : {
298 2 : steps.resize(steps.size() + 1);
299 : }
300 : }
301 399 : else if (arg.find("+gdal=") == 0)
302 : {
303 3 : const std::string stepName = arg.substr(strlen("+gdal="));
304 3 : curStep.alg = GetStepAlg(stepName);
305 3 : if (!curStep.alg)
306 : {
307 1 : ReportError(CE_Failure, CPLE_AppDefined,
308 : "unknown step name: %s", stepName.c_str());
309 1 : return false;
310 : }
311 : }
312 : #endif
313 396 : else if (!curStep.alg)
314 : {
315 184 : std::string algName = arg;
316 : #ifdef GDAL_PIPELINE_PROJ_NOSTALGIA
317 184 : if (!algName.empty() && algName[0] == '+')
318 1 : algName = algName.substr(1);
319 : #endif
320 184 : curStep.alg = GetStepAlg(algName);
321 184 : if (!curStep.alg)
322 : {
323 1 : ReportError(CE_Failure, CPLE_AppDefined,
324 : "unknown step name: %s", algName.c_str());
325 1 : return false;
326 : }
327 366 : curStep.alg->SetCallPath({std::move(algName)});
328 : }
329 : else
330 : {
331 212 : if (curStep.alg->HasSubAlgorithms())
332 : {
333 : auto subAlg = std::unique_ptr<GDALRasterPipelineStepAlgorithm>(
334 : cpl::down_cast<GDALRasterPipelineStepAlgorithm *>(
335 0 : curStep.alg->InstantiateSubAlgorithm(arg).release()));
336 0 : if (!subAlg)
337 : {
338 0 : ReportError(CE_Failure, CPLE_AppDefined,
339 : "'%s' is a unknown sub-algorithm of '%s'",
340 0 : arg.c_str(), curStep.alg->GetName().c_str());
341 0 : return false;
342 : }
343 0 : curStep.alg = std::move(subAlg);
344 0 : continue;
345 : }
346 :
347 : #ifdef GDAL_PIPELINE_PROJ_NOSTALGIA
348 216 : if (!arg.empty() && arg[0] == '+' &&
349 4 : arg.find(' ') == std::string::npos)
350 : {
351 3 : curStep.args.push_back("--" + arg.substr(1));
352 3 : continue;
353 : }
354 : #endif
355 209 : curStep.args.push_back(arg);
356 : }
357 : }
358 :
359 : // As we initially added a step without alg to bootstrap things, make
360 : // sure to remove it if it hasn't been filled, or the user has terminated
361 : // the pipeline with a '!' separator.
362 74 : if (!steps.back().alg)
363 2 : steps.pop_back();
364 :
365 : // Automatically add a final write step if none in m_executionForStreamOutput
366 : // mode
367 87 : if (m_executionForStreamOutput && !steps.empty() &&
368 13 : steps.back().alg->GetName() != GDALRasterWriteAlgorithm::NAME)
369 : {
370 11 : steps.resize(steps.size() + 1);
371 11 : steps.back().alg = GetStepAlg(GDALRasterWriteAlgorithm::NAME);
372 11 : steps.back().args.push_back("--output-format");
373 11 : steps.back().args.push_back("stream");
374 11 : steps.back().args.push_back("streamed_dataset");
375 : }
376 :
377 74 : bool helpRequested = false;
378 74 : if (IsCalledFromCommandLine())
379 : {
380 9 : for (auto &step : steps)
381 6 : step.alg->SetCalledFromCommandLine();
382 :
383 20 : for (const std::string &v : args)
384 : {
385 17 : if (cpl::ends_with(v, "=?"))
386 1 : helpRequested = true;
387 : }
388 : }
389 :
390 74 : if (steps.size() < 2)
391 : {
392 2 : if (!steps.empty() && helpRequested)
393 : {
394 0 : steps.back().alg->ParseCommandLineArguments(steps.back().args);
395 0 : return false;
396 : }
397 :
398 2 : ReportError(CE_Failure, CPLE_AppDefined,
399 : "At least 2 steps must be provided");
400 2 : return false;
401 : }
402 :
403 144 : std::vector<GDALRasterPipelineStepAlgorithm *> stepAlgs;
404 262 : for (const auto &step : steps)
405 190 : stepAlgs.push_back(step.alg.get());
406 72 : if (!CheckFirstStep(stepAlgs))
407 2 : return false;
408 :
409 70 : if (steps.back().alg->GetName() != GDALRasterWriteAlgorithm::NAME)
410 : {
411 2 : if (helpRequested)
412 : {
413 1 : steps.back().alg->ParseCommandLineArguments(steps.back().args);
414 1 : return false;
415 : }
416 1 : ReportError(CE_Failure, CPLE_AppDefined, "Last step should be '%s'",
417 : GDALRasterWriteAlgorithm::NAME);
418 1 : return false;
419 : }
420 180 : for (size_t i = 0; i < steps.size() - 1; ++i)
421 : {
422 113 : if (steps[i].alg->GetName() == GDALRasterWriteAlgorithm::NAME)
423 : {
424 1 : ReportError(CE_Failure, CPLE_AppDefined,
425 : "Only last step can be '%s'",
426 : GDALRasterWriteAlgorithm::NAME);
427 1 : return false;
428 : }
429 : }
430 :
431 245 : for (auto &step : steps)
432 : {
433 178 : step.alg->SetReferencePathForRelativePaths(
434 : GetReferencePathForRelativePaths());
435 : }
436 :
437 : // Propagate input parameters set at the pipeline level to the
438 : // "read" step
439 : {
440 67 : auto &step = steps.front();
441 562 : for (auto &arg : step.alg->GetArgs())
442 : {
443 495 : if (!arg->IsHidden())
444 : {
445 427 : auto pipelineArg = GetArg(arg->GetName());
446 427 : if (pipelineArg && pipelineArg->IsExplicitlySet())
447 : {
448 4 : arg->SetSkipIfAlreadySet(true);
449 4 : arg->SetFrom(*pipelineArg);
450 : }
451 : }
452 : }
453 : }
454 :
455 : // Same with "write" step
456 : {
457 67 : auto &step = steps.back();
458 670 : for (auto &arg : step.alg->GetArgs())
459 : {
460 603 : if (!arg->IsHidden())
461 : {
462 469 : auto pipelineArg = GetArg(arg->GetName());
463 469 : if (pipelineArg && pipelineArg->IsExplicitlySet())
464 : {
465 3 : arg->SetSkipIfAlreadySet(true);
466 3 : arg->SetFrom(*pipelineArg);
467 : }
468 : }
469 : }
470 : }
471 :
472 : // Parse each step, but without running the validation
473 234 : for (const auto &step : steps)
474 : {
475 173 : step.alg->m_skipValidationInParseCommandLine = true;
476 173 : if (!step.alg->ParseCommandLineArguments(step.args))
477 6 : return false;
478 : }
479 :
480 : // Evaluate "input" argument of "read" step, together with the "output"
481 : // argument of the "write" step, in case they point to the same dataset.
482 61 : auto inputArg = steps.front().alg->GetArg(GDAL_ARG_NAME_INPUT);
483 122 : if (inputArg && inputArg->IsExplicitlySet() &&
484 183 : inputArg->GetType() == GAAT_DATASET_LIST &&
485 61 : inputArg->Get<std::vector<GDALArgDatasetValue>>().size() == 1)
486 : {
487 59 : steps.front().alg->ProcessDatasetArg(inputArg, steps.back().alg.get());
488 : }
489 :
490 216 : for (const auto &step : steps)
491 : {
492 159 : if (!step.alg->ValidateArguments())
493 4 : return false;
494 : }
495 :
496 209 : for (auto &step : steps)
497 152 : m_steps.push_back(std::move(step.alg));
498 :
499 57 : return true;
500 : }
501 :
502 : /************************************************************************/
503 : /* GDALRasterPipelineAlgorithm::GetUsageForCLI() */
504 : /************************************************************************/
505 :
506 8 : std::string GDALRasterPipelineAlgorithm::GetUsageForCLI(
507 : bool shortUsage, const UsageOptions &usageOptions) const
508 : {
509 8 : UsageOptions stepUsageOptions;
510 8 : stepUsageOptions.isPipelineStep = true;
511 :
512 8 : if (!m_helpDocCategory.empty() && m_helpDocCategory != "main")
513 : {
514 4 : auto alg = GetStepAlg(m_helpDocCategory);
515 4 : std::string ret;
516 2 : if (alg)
517 : {
518 2 : alg->SetCallPath({m_helpDocCategory});
519 1 : alg->GetArg("help-doc")->Set(true);
520 1 : return alg->GetUsageForCLI(shortUsage, stepUsageOptions);
521 : }
522 : else
523 : {
524 1 : fprintf(stderr, "ERROR: unknown pipeline step '%s'\n",
525 : m_helpDocCategory.c_str());
526 : return CPLSPrintf("ERROR: unknown pipeline step '%s'\n",
527 1 : m_helpDocCategory.c_str());
528 : }
529 : }
530 :
531 6 : UsageOptions usageOptionsMain(usageOptions);
532 6 : usageOptionsMain.isPipelineMain = true;
533 : std::string ret =
534 12 : GDALAlgorithm::GetUsageForCLI(shortUsage, usageOptionsMain);
535 6 : if (shortUsage)
536 2 : return ret;
537 :
538 : ret += "\n<PIPELINE> is of the form: read|mosaic|stack [READ-OPTIONS] "
539 4 : "( ! <STEP-NAME> [STEP-OPTIONS] )* ! write [WRITE-OPTIONS]\n";
540 :
541 4 : if (m_helpDocCategory == "main")
542 : {
543 1 : return ret;
544 : }
545 :
546 3 : ret += '\n';
547 3 : ret += "Example: 'gdal raster pipeline --progress ! read in.tif ! \\\n";
548 3 : ret += " reproject --dst-crs=EPSG:32632 ! ";
549 3 : ret += "write out.tif --overwrite'\n";
550 3 : ret += '\n';
551 3 : ret += "Potential steps are:\n";
552 :
553 90 : for (const std::string &name : m_stepRegistry.GetNames())
554 : {
555 174 : auto alg = GetStepAlg(name);
556 87 : auto [options, maxOptLen] = alg->GetArgNamesForCLI();
557 87 : stepUsageOptions.maxOptLen =
558 87 : std::max(stepUsageOptions.maxOptLen, maxOptLen);
559 : }
560 :
561 : {
562 3 : const auto name = GDALRasterReadAlgorithm::NAME;
563 3 : ret += '\n';
564 6 : auto alg = GetStepAlg(name);
565 6 : alg->SetCallPath({name});
566 3 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
567 : }
568 90 : for (const std::string &name : m_stepRegistry.GetNames())
569 : {
570 174 : auto alg = GetStepAlg(name);
571 87 : assert(alg);
572 99 : if (alg->CanBeFirstStep() && !alg->IsHidden() &&
573 12 : name != GDALRasterReadAlgorithm::NAME)
574 : {
575 9 : ret += '\n';
576 18 : alg->SetCallPath({name});
577 9 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
578 : }
579 : }
580 90 : for (const std::string &name : m_stepRegistry.GetNames())
581 : {
582 174 : auto alg = GetStepAlg(name);
583 87 : assert(alg);
584 162 : if (!alg->CanBeFirstStep() && !alg->IsHidden() &&
585 75 : name != GDALRasterWriteAlgorithm::NAME)
586 : {
587 72 : ret += '\n';
588 144 : alg->SetCallPath({name});
589 72 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
590 : }
591 : }
592 : {
593 3 : const auto name = GDALRasterWriteAlgorithm::NAME;
594 3 : ret += '\n';
595 6 : auto alg = GetStepAlg(name);
596 6 : alg->SetCallPath({name});
597 3 : ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
598 : }
599 :
600 3 : ret += GetUsageForCLIEnd();
601 :
602 3 : return ret;
603 : }
604 :
605 : /************************************************************************/
606 : /* GDALRasterPipelineNonNativelyStreamingAlgorithm() */
607 : /************************************************************************/
608 :
609 189 : GDALRasterPipelineNonNativelyStreamingAlgorithm::
610 : GDALRasterPipelineNonNativelyStreamingAlgorithm(
611 : const std::string &name, const std::string &description,
612 189 : const std::string &helpURL, bool standaloneStep)
613 : : GDALRasterPipelineStepAlgorithm(name, description, helpURL,
614 189 : standaloneStep)
615 : {
616 189 : }
617 :
618 : /************************************************************************/
619 : /* IsNativelyStreamingCompatible() */
620 : /************************************************************************/
621 :
622 41 : bool GDALRasterPipelineNonNativelyStreamingAlgorithm::
623 : IsNativelyStreamingCompatible() const
624 : {
625 41 : return false;
626 : }
627 :
628 : /************************************************************************/
629 : /* MustCreateOnDiskTempDataset() */
630 : /************************************************************************/
631 :
632 55 : static bool MustCreateOnDiskTempDataset(int nWidth, int nHeight, int nBands,
633 : GDALDataType eDT)
634 : {
635 : // Config option mostly for autotest purposes
636 55 : if (CPLTestBool(CPLGetConfigOption(
637 : "GDAL_RASTER_PIPELINE_USE_GTIFF_FOR_TEMP_DATASET", "NO")))
638 5 : return true;
639 :
640 : // Allow up to 10% of RAM usage for temporary dataset
641 50 : const auto nRAM = CPLGetUsablePhysicalRAM() / 10;
642 50 : const int nDTSize = GDALGetDataTypeSizeBytes(eDT);
643 50 : const bool bOnDisk =
644 100 : nBands > 0 && nDTSize > 0 && nRAM > 0 &&
645 50 : static_cast<int64_t>(nWidth) * nHeight > nRAM / (nBands * nDTSize);
646 50 : return bOnDisk;
647 : }
648 :
649 : /************************************************************************/
650 : /* CreateTemporaryDataset() */
651 : /************************************************************************/
652 :
653 : std::unique_ptr<GDALDataset>
654 30 : GDALRasterPipelineNonNativelyStreamingAlgorithm::CreateTemporaryDataset(
655 : int nWidth, int nHeight, int nBands, GDALDataType eDT,
656 : bool bTiledIfPossible, GDALDataset *poSrcDSForMetadata, bool bCopyMetadata)
657 : {
658 : const bool bOnDisk =
659 30 : MustCreateOnDiskTempDataset(nWidth, nHeight, nBands, eDT);
660 30 : const char *pszDriverName = bOnDisk ? "GTIFF" : "MEM";
661 : GDALDriver *poDriver =
662 30 : GetGDALDriverManager()->GetDriverByName(pszDriverName);
663 60 : CPLStringList aosOptions;
664 60 : std::string osTmpFilename;
665 30 : if (bOnDisk)
666 : {
667 : osTmpFilename =
668 4 : CPLGenerateTempFilenameSafe(
669 : poSrcDSForMetadata
670 4 : ? CPLGetBasenameSafe(poSrcDSForMetadata->GetDescription())
671 2 : .c_str()
672 4 : : "") +
673 2 : ".tif";
674 2 : if (bTiledIfPossible)
675 2 : aosOptions.SetNameValue("TILED", "YES");
676 : const char *pszCOList =
677 2 : poDriver->GetMetadataItem(GDAL_DMD_CREATIONOPTIONLIST);
678 : aosOptions.SetNameValue("COMPRESS",
679 2 : pszCOList && strstr(pszCOList, "ZSTD") ? "ZSTD"
680 4 : : "LZW");
681 2 : aosOptions.SetNameValue("SPARSE_OK", "YES");
682 : }
683 : std::unique_ptr<GDALDataset> poOutDS(
684 30 : poDriver ? poDriver->Create(osTmpFilename.c_str(), nWidth, nHeight,
685 30 : nBands, eDT, aosOptions.List())
686 60 : : nullptr);
687 30 : if (poOutDS && bOnDisk)
688 : {
689 : // In file systems that allow it (all but Windows...), we want to
690 : // delete the temporary file as soon as soon as possible after
691 : // having open it, so that if someone kills the process there are
692 : // no temp files left over. If that unlink() doesn't succeed
693 : // (on Windows), then the file will eventually be deleted when
694 : // poTmpDS is cleaned due to MarkSuppressOnClose().
695 0 : VSIUnlink(osTmpFilename.c_str());
696 0 : poOutDS->MarkSuppressOnClose();
697 : }
698 :
699 30 : if (poOutDS && poSrcDSForMetadata)
700 : {
701 28 : poOutDS->SetSpatialRef(poSrcDSForMetadata->GetSpatialRef());
702 28 : std::array<double, 6> adfGT{};
703 28 : if (poSrcDSForMetadata->GetGeoTransform(adfGT.data()) == CE_None)
704 27 : poOutDS->SetGeoTransform(adfGT.data());
705 28 : if (const int nGCPCount = poSrcDSForMetadata->GetGCPCount())
706 : {
707 0 : const auto apsGCPs = poSrcDSForMetadata->GetGCPs();
708 0 : if (apsGCPs)
709 : {
710 0 : poOutDS->SetGCPs(nGCPCount, apsGCPs,
711 0 : poSrcDSForMetadata->GetGCPSpatialRef());
712 : }
713 : }
714 28 : if (bCopyMetadata)
715 : {
716 14 : poOutDS->SetMetadata(poSrcDSForMetadata->GetMetadata());
717 : }
718 : }
719 :
720 60 : return poOutDS;
721 : }
722 :
723 : /************************************************************************/
724 : /* CreateTemporaryCopy() */
725 : /************************************************************************/
726 :
727 : std::unique_ptr<GDALDataset>
728 25 : GDALRasterPipelineNonNativelyStreamingAlgorithm::CreateTemporaryCopy(
729 : GDALAlgorithm *poAlg, GDALDataset *poSrcDS, int nSingleBand,
730 : bool bTiledIfPossible, GDALProgressFunc pfnProgress, void *pProgressData)
731 : {
732 25 : const int nBands = nSingleBand > 0 ? 1 : poSrcDS->GetRasterCount();
733 : const auto eDT =
734 25 : nBands ? poSrcDS->GetRasterBand(1)->GetRasterDataType() : GDT_Unknown;
735 25 : const bool bOnDisk = MustCreateOnDiskTempDataset(
736 : poSrcDS->GetRasterXSize(), poSrcDS->GetRasterYSize(), nBands, eDT);
737 25 : const char *pszDriverName = bOnDisk ? "GTIFF" : "MEM";
738 :
739 50 : CPLStringList options;
740 25 : if (nSingleBand > 0)
741 : {
742 25 : options.AddString("-b");
743 25 : options.AddString(CPLSPrintf("%d", nSingleBand));
744 : }
745 :
746 25 : options.AddString("-of");
747 25 : options.AddString(pszDriverName);
748 :
749 50 : std::string osTmpFilename;
750 25 : if (bOnDisk)
751 : {
752 : osTmpFilename =
753 3 : CPLGenerateTempFilenameSafe(
754 9 : CPLGetBasenameSafe(poSrcDS->GetDescription()).c_str()) +
755 3 : ".tif";
756 3 : if (bTiledIfPossible)
757 : {
758 3 : options.AddString("-co");
759 3 : options.AddString("TILED=YES");
760 : }
761 :
762 : GDALDriver *poDriver =
763 3 : GetGDALDriverManager()->GetDriverByName(pszDriverName);
764 : const char *pszCOList =
765 3 : poDriver ? poDriver->GetMetadataItem(GDAL_DMD_CREATIONOPTIONLIST)
766 3 : : nullptr;
767 3 : options.AddString("-co");
768 3 : options.AddString(pszCOList && strstr(pszCOList, "ZSTD")
769 : ? "COMPRESS=ZSTD"
770 6 : : "COMPRESS=LZW");
771 : }
772 :
773 : GDALTranslateOptions *translateOptions =
774 25 : GDALTranslateOptionsNew(options.List(), nullptr);
775 :
776 25 : if (pfnProgress)
777 12 : GDALTranslateOptionsSetProgress(translateOptions, pfnProgress,
778 : pProgressData);
779 :
780 : std::unique_ptr<GDALDataset> poOutDS(GDALDataset::FromHandle(
781 : GDALTranslate(osTmpFilename.c_str(), GDALDataset::ToHandle(poSrcDS),
782 25 : translateOptions, nullptr)));
783 25 : GDALTranslateOptionsFree(translateOptions);
784 :
785 25 : if (!poOutDS)
786 : {
787 2 : poAlg->ReportError(CE_Failure, CPLE_AppDefined,
788 : "Failed to create temporary dataset");
789 : }
790 23 : else if (bOnDisk)
791 : {
792 : // In file systems that allow it (all but Windows...), we want to
793 : // delete the temporary file as soon as soon as possible after
794 : // having open it, so that if someone kills the process there are
795 : // no temp files left over. If that unlink() doesn't succeed
796 : // (on Windows), then the file will eventually be deleted when
797 : // poTmpDS is cleaned due to MarkSuppressOnClose().
798 1 : VSIUnlink(osTmpFilename.c_str());
799 1 : poOutDS->MarkSuppressOnClose();
800 : }
801 50 : return poOutDS;
802 : }
803 :
804 : //! @endcond
|
