LCOV - code coverage report
Current view: top level - apps - gdalalg_raster_pipeline.cpp (source / functions) Hit Total Coverage
Test: gdal_filtered.info Lines: 352 368 95.7 %
Date: 2025-07-09 17:50:03 Functions: 19 19 100.0 %

          Line data    Source code
       1             : /******************************************************************************
       2             :  *
       3             :  * Project:  GDAL
       4             :  * Purpose:  gdal "raster pipeline" subcommand
       5             :  * Author:   Even Rouault <even dot rouault at spatialys.com>
       6             :  *
       7             :  ******************************************************************************
       8             :  * Copyright (c) 2024, Even Rouault <even dot rouault at spatialys.com>
       9             :  *
      10             :  * SPDX-License-Identifier: MIT
      11             :  ****************************************************************************/
      12             : 
      13             : #include "gdalalg_raster_pipeline.h"
      14             : #include "gdalalg_raster_read.h"
      15             : #include "gdalalg_raster_calc.h"
      16             : #include "gdalalg_raster_aspect.h"
      17             : #include "gdalalg_raster_clip.h"
      18             : #include "gdalalg_raster_color_map.h"
      19             : #include "gdalalg_raster_color_merge.h"
      20             : #include "gdalalg_raster_edit.h"
      21             : #include "gdalalg_raster_fill_nodata.h"
      22             : #include "gdalalg_raster_hillshade.h"
      23             : #include "gdalalg_raster_mosaic.h"
      24             : #include "gdalalg_raster_nodata_to_alpha.h"
      25             : #include "gdalalg_raster_pansharpen.h"
      26             : #include "gdalalg_raster_proximity.h"
      27             : #include "gdalalg_raster_reclassify.h"
      28             : #include "gdalalg_raster_reproject.h"
      29             : #include "gdalalg_raster_resize.h"
      30             : #include "gdalalg_raster_rgb_to_palette.h"
      31             : #include "gdalalg_raster_roughness.h"
      32             : #include "gdalalg_raster_scale.h"
      33             : #include "gdalalg_raster_select.h"
      34             : #include "gdalalg_raster_set_type.h"
      35             : #include "gdalalg_raster_sieve.h"
      36             : #include "gdalalg_raster_slope.h"
      37             : #include "gdalalg_raster_stack.h"
      38             : #include "gdalalg_raster_write.h"
      39             : #include "gdalalg_raster_tpi.h"
      40             : #include "gdalalg_raster_tri.h"
      41             : #include "gdalalg_raster_unscale.h"
      42             : #include "gdalalg_raster_viewshed.h"
      43             : 
      44             : #include "cpl_conv.h"
      45             : #include "cpl_progress.h"
      46             : #include "cpl_string.h"
      47             : #include "cpl_vsi.h"
      48             : #include "gdal_priv.h"
      49             : #include "gdal_utils.h"
      50             : 
      51             : #include <algorithm>
      52             : #include <array>
      53             : #include <cassert>
      54             : 
      55             : //! @cond Doxygen_Suppress
      56             : 
      57             : #ifndef _
      58             : #define _(x) (x)
      59             : #endif
      60             : 
      61             : /************************************************************************/
      62             : /*  GDALRasterPipelineStepAlgorithm::GDALRasterPipelineStepAlgorithm()  */
      63             : /************************************************************************/
      64             : 
      65        1193 : GDALRasterPipelineStepAlgorithm::GDALRasterPipelineStepAlgorithm(
      66             :     const std::string &name, const std::string &description,
      67        1193 :     const std::string &helpURL, bool standaloneStep)
      68             :     : GDALRasterPipelineStepAlgorithm(
      69             :           name, description, helpURL,
      70        1193 :           ConstructorOptions().SetStandaloneStep(standaloneStep))
      71             : {
      72        1193 : }
      73             : 
      74             : /************************************************************************/
      75             : /*  GDALRasterPipelineStepAlgorithm::GDALRasterPipelineStepAlgorithm()  */
      76             : /************************************************************************/
      77             : 
      78        2007 : GDALRasterPipelineStepAlgorithm::GDALRasterPipelineStepAlgorithm(
      79             :     const std::string &name, const std::string &description,
      80        2007 :     const std::string &helpURL, const ConstructorOptions &options)
      81        2007 :     : GDALPipelineStepAlgorithm(name, description, helpURL, options)
      82             : {
      83        2007 :     if (m_standaloneStep)
      84             :     {
      85         501 :         m_supportsStreamedOutput = true;
      86             : 
      87         501 :         if (m_constructorOptions.addDefaultArguments)
      88             :         {
      89         332 :             AddRasterInputArgs(false, false);
      90         332 :             AddProgressArg();
      91         332 :             AddRasterOutputArgs(false);
      92             :         }
      93             :     }
      94        1506 :     else if (m_constructorOptions.addDefaultArguments)
      95             :     {
      96         861 :         AddRasterHiddenInputDatasetArg();
      97             :     }
      98        2007 : }
      99             : 
     100             : GDALRasterPipelineStepAlgorithm::~GDALRasterPipelineStepAlgorithm() = default;
     101             : 
     102             : /************************************************************************/
     103             : /*        GDALRasterPipelineStepAlgorithm::SetOutputVRTCompatible()     */
     104             : /************************************************************************/
     105             : 
     106         219 : void GDALRasterPipelineStepAlgorithm::SetOutputVRTCompatible(bool b)
     107             : {
     108         219 :     m_outputVRTCompatible = b;
     109         219 :     if (m_outputFormatArg)
     110             :     {
     111          91 :         m_outputFormatArg->AddMetadataItem(GAAMDI_VRT_COMPATIBLE,
     112         182 :                                            {b ? "true" : "false"});
     113             :     }
     114         219 : }
     115             : 
     116             : /************************************************************************/
     117             : /*        GDALRasterPipelineAlgorithm::GDALRasterPipelineAlgorithm()    */
     118             : /************************************************************************/
     119             : 
     120          99 : GDALRasterPipelineAlgorithm::GDALRasterPipelineAlgorithm(
     121          99 :     bool openForMixedRasterVector)
     122             :     : GDALAbstractPipelineAlgorithm<GDALRasterPipelineStepAlgorithm>(
     123             :           NAME, DESCRIPTION, HELP_URL,
     124           0 :           ConstructorOptions()
     125          99 :               .SetAddDefaultArguments(false)
     126         198 :               .SetInputDatasetMaxCount(INT_MAX))
     127             : {
     128          99 :     m_supportsStreamedOutput = true;
     129             : 
     130          99 :     AddRasterInputArgs(openForMixedRasterVector, /* hiddenForCLI = */ true);
     131          99 :     AddProgressArg();
     132         198 :     AddArg("pipeline", 0, _("Pipeline string"), &m_pipeline)
     133          99 :         .SetHiddenForCLI()
     134          99 :         .SetPositional();
     135          99 :     AddRasterOutputArgs(/* hiddenForCLI = */ true);
     136             : 
     137          99 :     RegisterAlgorithms(m_stepRegistry, false);
     138          99 : }
     139             : 
     140             : /************************************************************************/
     141             : /*       GDALRasterPipelineAlgorithm::RegisterAlgorithms()              */
     142             : /************************************************************************/
     143             : 
     144             : /* static */
     145         148 : void GDALRasterPipelineAlgorithm::RegisterAlgorithms(
     146             :     GDALAlgorithmRegistry &registry, bool forMixedPipeline)
     147             : {
     148         296 :     GDALAlgorithmRegistry::AlgInfo algInfo;
     149             : 
     150             :     const auto addSuffixIfNeeded =
     151         888 :         [forMixedPipeline](const char *name) -> std::string
     152             :     {
     153        1182 :         return forMixedPipeline ? std::string(name).append(RASTER_SUFFIX)
     154        2070 :                                 : std::string(name);
     155         148 :     };
     156             : 
     157         148 :     algInfo.m_name = addSuffixIfNeeded(GDALRasterReadAlgorithm::NAME);
     158          98 :     algInfo.m_creationFunc = []() -> std::unique_ptr<GDALAlgorithm>
     159         246 :     { return std::make_unique<GDALRasterReadAlgorithm>(); };
     160         148 :     registry.Register(algInfo);
     161             : 
     162         148 :     registry.Register<GDALRasterCalcAlgorithm>();
     163             : 
     164         148 :     algInfo.m_name = addSuffixIfNeeded(GDALRasterWriteAlgorithm::NAME);
     165         130 :     algInfo.m_creationFunc = []() -> std::unique_ptr<GDALAlgorithm>
     166         278 :     { return std::make_unique<GDALRasterWriteAlgorithm>(); };
     167         148 :     registry.Register(algInfo);
     168             : 
     169         148 :     registry.Register<GDALRasterAspectAlgorithm>();
     170             : 
     171         148 :     algInfo.m_name = addSuffixIfNeeded(GDALRasterClipAlgorithm::NAME);
     172          19 :     algInfo.m_creationFunc = []() -> std::unique_ptr<GDALAlgorithm>
     173         167 :     { return std::make_unique<GDALRasterClipAlgorithm>(); };
     174         148 :     registry.Register(algInfo);
     175             : 
     176         148 :     registry.Register<GDALRasterColorMapAlgorithm>();
     177         148 :     registry.Register<GDALRasterColorMergeAlgorithm>();
     178             : 
     179         148 :     algInfo.m_name = addSuffixIfNeeded(GDALRasterEditAlgorithm::NAME);
     180          33 :     algInfo.m_creationFunc = []() -> std::unique_ptr<GDALAlgorithm>
     181         181 :     { return std::make_unique<GDALRasterEditAlgorithm>(); };
     182         148 :     registry.Register(algInfo);
     183             : 
     184         148 :     registry.Register<GDALRasterNoDataToAlphaAlgorithm>();
     185         148 :     registry.Register<GDALRasterFillNodataAlgorithm>();
     186         148 :     registry.Register<GDALRasterHillshadeAlgorithm>();
     187         148 :     registry.Register<GDALRasterMosaicAlgorithm>();
     188         148 :     registry.Register<GDALRasterPansharpenAlgorithm>();
     189         148 :     registry.Register<GDALRasterProximityAlgorithm>();
     190         148 :     registry.Register<GDALRasterReclassifyAlgorithm>();
     191             : 
     192         148 :     algInfo.m_name = addSuffixIfNeeded(GDALRasterReprojectAlgorithm::NAME);
     193          41 :     algInfo.m_creationFunc = []() -> std::unique_ptr<GDALAlgorithm>
     194         189 :     { return std::make_unique<GDALRasterReprojectAlgorithm>(); };
     195         148 :     registry.Register(algInfo);
     196             : 
     197         148 :     registry.Register<GDALRasterResizeAlgorithm>();
     198         148 :     registry.Register<GDALRasterRGBToPaletteAlgorithm>();
     199         148 :     registry.Register<GDALRasterRoughnessAlgorithm>();
     200         148 :     registry.Register<GDALRasterScaleAlgorithm>();
     201             : 
     202         148 :     algInfo.m_name = addSuffixIfNeeded(GDALRasterSelectAlgorithm::NAME);
     203          20 :     algInfo.m_creationFunc = []() -> std::unique_ptr<GDALAlgorithm>
     204         168 :     { return std::make_unique<GDALRasterSelectAlgorithm>(); };
     205         148 :     registry.Register(algInfo);
     206             : 
     207         148 :     registry.Register<GDALRasterSetTypeAlgorithm>();
     208         148 :     registry.Register<GDALRasterSieveAlgorithm>();
     209         148 :     registry.Register<GDALRasterSlopeAlgorithm>();
     210         148 :     registry.Register<GDALRasterStackAlgorithm>();
     211         148 :     registry.Register<GDALRasterTPIAlgorithm>();
     212         148 :     registry.Register<GDALRasterTRIAlgorithm>();
     213         148 :     registry.Register<GDALRasterUnscaleAlgorithm>();
     214         148 :     registry.Register<GDALRasterViewshedAlgorithm>();
     215         148 : }
     216             : 
     217             : /************************************************************************/
     218             : /*       GDALRasterPipelineAlgorithm::ParseCommandLineArguments()       */
     219             : /************************************************************************/
     220             : 
     221          87 : bool GDALRasterPipelineAlgorithm::ParseCommandLineArguments(
     222             :     const std::vector<std::string> &args)
     223             : {
     224          99 :     if (args.size() == 1 && (args[0] == "-h" || args[0] == "--help" ||
     225          12 :                              args[0] == "help" || args[0] == "--json-usage"))
     226             :     {
     227           2 :         return GDALAlgorithm::ParseCommandLineArguments(args);
     228             :     }
     229          85 :     else if (args.size() == 1 && STARTS_WITH(args[0].c_str(), "--help-doc="))
     230             :     {
     231           3 :         m_helpDocCategory = args[0].substr(strlen("--help-doc="));
     232           6 :         return GDALAlgorithm::ParseCommandLineArguments({"--help-doc"});
     233             :     }
     234             : 
     235         620 :     for (const auto &arg : args)
     236             :     {
     237         540 :         if (arg.find("--pipeline") == 0)
     238           2 :             return GDALAlgorithm::ParseCommandLineArguments(args);
     239             : 
     240             :         // gdal raster pipeline [--progress] "read in.tif ..."
     241         539 :         if (arg.find("read ") == 0)
     242           1 :             return GDALAlgorithm::ParseCommandLineArguments(args);
     243             :     }
     244             : 
     245          80 :     if (!m_steps.empty())
     246             :     {
     247           1 :         ReportError(CE_Failure, CPLE_AppDefined,
     248             :                     "ParseCommandLineArguments() can only be called once per "
     249             :                     "instance.");
     250           1 :         return false;
     251             :     }
     252             : 
     253             :     struct Step
     254             :     {
     255             :         std::unique_ptr<GDALRasterPipelineStepAlgorithm> alg{};
     256             :         std::vector<std::string> args{};
     257             :     };
     258             : 
     259         158 :     std::vector<Step> steps;
     260          79 :     steps.resize(1);
     261             : 
     262         601 :     for (const auto &arg : args)
     263             :     {
     264         527 :         if (arg == "--progress")
     265             :         {
     266           2 :             m_progressBarRequested = true;
     267           2 :             continue;
     268             :         }
     269             : 
     270         525 :         if (IsCalledFromCommandLine() && (arg == "-h" || arg == "--help"))
     271             :         {
     272           3 :             if (!steps.back().alg)
     273           1 :                 steps.pop_back();
     274           3 :             if (steps.empty())
     275             :             {
     276           5 :                 return GDALAlgorithm::ParseCommandLineArguments(args);
     277             :             }
     278             :             else
     279             :             {
     280           2 :                 m_stepOnWhichHelpIsRequested = std::move(steps.back().alg);
     281           2 :                 return true;
     282             :             }
     283             :         }
     284             : 
     285         522 :         auto &curStep = steps.back();
     286             : 
     287         522 :         if (arg == "!" || arg == "|")
     288             :         {
     289         119 :             if (curStep.alg)
     290             :             {
     291         109 :                 steps.resize(steps.size() + 1);
     292             :             }
     293             :         }
     294             : #ifdef GDAL_PIPELINE_PROJ_NOSTALGIA
     295         403 :         else if (arg == "+step")
     296             :         {
     297           4 :             if (curStep.alg)
     298             :             {
     299           2 :                 steps.resize(steps.size() + 1);
     300             :             }
     301             :         }
     302         399 :         else if (arg.find("+gdal=") == 0)
     303             :         {
     304           3 :             const std::string stepName = arg.substr(strlen("+gdal="));
     305           3 :             curStep.alg = GetStepAlg(stepName);
     306           3 :             if (!curStep.alg)
     307             :             {
     308           1 :                 ReportError(CE_Failure, CPLE_AppDefined,
     309             :                             "unknown step name: %s", stepName.c_str());
     310           1 :                 return false;
     311             :             }
     312             :         }
     313             : #endif
     314         396 :         else if (!curStep.alg)
     315             :         {
     316         184 :             std::string algName = arg;
     317             : #ifdef GDAL_PIPELINE_PROJ_NOSTALGIA
     318         184 :             if (!algName.empty() && algName[0] == '+')
     319           1 :                 algName = algName.substr(1);
     320             : #endif
     321         184 :             curStep.alg = GetStepAlg(algName);
     322         184 :             if (!curStep.alg)
     323             :             {
     324           1 :                 ReportError(CE_Failure, CPLE_AppDefined,
     325             :                             "unknown step name: %s", algName.c_str());
     326           1 :                 return false;
     327             :             }
     328         366 :             curStep.alg->SetCallPath({std::move(algName)});
     329             :         }
     330             :         else
     331             :         {
     332         212 :             if (curStep.alg->HasSubAlgorithms())
     333             :             {
     334             :                 auto subAlg = std::unique_ptr<GDALRasterPipelineStepAlgorithm>(
     335             :                     cpl::down_cast<GDALRasterPipelineStepAlgorithm *>(
     336           0 :                         curStep.alg->InstantiateSubAlgorithm(arg).release()));
     337           0 :                 if (!subAlg)
     338             :                 {
     339           0 :                     ReportError(CE_Failure, CPLE_AppDefined,
     340             :                                 "'%s' is a unknown sub-algorithm of '%s'",
     341           0 :                                 arg.c_str(), curStep.alg->GetName().c_str());
     342           0 :                     return false;
     343             :                 }
     344           0 :                 curStep.alg = std::move(subAlg);
     345           0 :                 continue;
     346             :             }
     347             : 
     348             : #ifdef GDAL_PIPELINE_PROJ_NOSTALGIA
     349         216 :             if (!arg.empty() && arg[0] == '+' &&
     350           4 :                 arg.find(' ') == std::string::npos)
     351             :             {
     352           3 :                 curStep.args.push_back("--" + arg.substr(1));
     353           3 :                 continue;
     354             :             }
     355             : #endif
     356         209 :             curStep.args.push_back(arg);
     357             :         }
     358             :     }
     359             : 
     360             :     // As we initially added a step without alg to bootstrap things, make
     361             :     // sure to remove it if it hasn't been filled, or the user has terminated
     362             :     // the pipeline with a '!' separator.
     363          74 :     if (!steps.back().alg)
     364           2 :         steps.pop_back();
     365             : 
     366             :     // Automatically add a final write step if none in m_executionForStreamOutput
     367             :     // mode
     368          87 :     if (m_executionForStreamOutput && !steps.empty() &&
     369          13 :         steps.back().alg->GetName() != GDALRasterWriteAlgorithm::NAME)
     370             :     {
     371          11 :         steps.resize(steps.size() + 1);
     372          11 :         steps.back().alg = GetStepAlg(GDALRasterWriteAlgorithm::NAME);
     373          11 :         steps.back().args.push_back("--output-format");
     374          11 :         steps.back().args.push_back("stream");
     375          11 :         steps.back().args.push_back("streamed_dataset");
     376             :     }
     377             : 
     378          74 :     bool helpRequested = false;
     379          74 :     if (IsCalledFromCommandLine())
     380             :     {
     381           9 :         for (auto &step : steps)
     382           6 :             step.alg->SetCalledFromCommandLine();
     383             : 
     384          20 :         for (const std::string &v : args)
     385             :         {
     386          17 :             if (cpl::ends_with(v, "=?"))
     387           1 :                 helpRequested = true;
     388             :         }
     389             :     }
     390             : 
     391          74 :     if (steps.size() < 2)
     392             :     {
     393           2 :         if (!steps.empty() && helpRequested)
     394             :         {
     395           0 :             steps.back().alg->ParseCommandLineArguments(steps.back().args);
     396           0 :             return false;
     397             :         }
     398             : 
     399           2 :         ReportError(CE_Failure, CPLE_AppDefined,
     400             :                     "At least 2 steps must be provided");
     401           2 :         return false;
     402             :     }
     403             : 
     404         144 :     std::vector<GDALRasterPipelineStepAlgorithm *> stepAlgs;
     405         262 :     for (const auto &step : steps)
     406         190 :         stepAlgs.push_back(step.alg.get());
     407          72 :     if (!CheckFirstStep(stepAlgs))
     408           2 :         return false;
     409             : 
     410          70 :     if (steps.back().alg->GetName() != GDALRasterWriteAlgorithm::NAME)
     411             :     {
     412           2 :         if (helpRequested)
     413             :         {
     414           1 :             steps.back().alg->ParseCommandLineArguments(steps.back().args);
     415           1 :             return false;
     416             :         }
     417           1 :         ReportError(CE_Failure, CPLE_AppDefined, "Last step should be '%s'",
     418             :                     GDALRasterWriteAlgorithm::NAME);
     419           1 :         return false;
     420             :     }
     421         180 :     for (size_t i = 0; i < steps.size() - 1; ++i)
     422             :     {
     423         113 :         if (steps[i].alg->GetName() == GDALRasterWriteAlgorithm::NAME)
     424             :         {
     425           1 :             ReportError(CE_Failure, CPLE_AppDefined,
     426             :                         "Only last step can be '%s'",
     427             :                         GDALRasterWriteAlgorithm::NAME);
     428           1 :             return false;
     429             :         }
     430             :     }
     431             : 
     432         245 :     for (auto &step : steps)
     433             :     {
     434         178 :         step.alg->SetReferencePathForRelativePaths(
     435             :             GetReferencePathForRelativePaths());
     436             :     }
     437             : 
     438             :     // Propagate input parameters set at the pipeline level to the
     439             :     // "read" step
     440             :     {
     441          67 :         auto &step = steps.front();
     442         564 :         for (auto &arg : step.alg->GetArgs())
     443             :         {
     444         497 :             if (!arg->IsHidden())
     445             :             {
     446         429 :                 auto pipelineArg = GetArg(arg->GetName());
     447         429 :                 if (pipelineArg && pipelineArg->IsExplicitlySet())
     448             :                 {
     449           4 :                     arg->SetSkipIfAlreadySet(true);
     450           4 :                     arg->SetFrom(*pipelineArg);
     451             :                 }
     452             :             }
     453             :         }
     454             :     }
     455             : 
     456             :     // Same with "write" step
     457             :     {
     458          67 :         auto &step = steps.back();
     459         670 :         for (auto &arg : step.alg->GetArgs())
     460             :         {
     461         603 :             if (!arg->IsHidden())
     462             :             {
     463         469 :                 auto pipelineArg = GetArg(arg->GetName());
     464         469 :                 if (pipelineArg && pipelineArg->IsExplicitlySet())
     465             :                 {
     466           3 :                     arg->SetSkipIfAlreadySet(true);
     467           3 :                     arg->SetFrom(*pipelineArg);
     468             :                 }
     469             :             }
     470             :         }
     471             :     }
     472             : 
     473             :     // Parse each step, but without running the validation
     474         234 :     for (const auto &step : steps)
     475             :     {
     476         173 :         step.alg->m_skipValidationInParseCommandLine = true;
     477         173 :         if (!step.alg->ParseCommandLineArguments(step.args))
     478           6 :             return false;
     479             :     }
     480             : 
     481             :     // Evaluate "input" argument of "read" step, together with the "output"
     482             :     // argument of the "write" step, in case they point to the same dataset.
     483          61 :     auto inputArg = steps.front().alg->GetArg(GDAL_ARG_NAME_INPUT);
     484         122 :     if (inputArg && inputArg->IsExplicitlySet() &&
     485         183 :         inputArg->GetType() == GAAT_DATASET_LIST &&
     486          61 :         inputArg->Get<std::vector<GDALArgDatasetValue>>().size() == 1)
     487             :     {
     488          59 :         steps.front().alg->ProcessDatasetArg(inputArg, steps.back().alg.get());
     489             :     }
     490             : 
     491         216 :     for (const auto &step : steps)
     492             :     {
     493         159 :         if (!step.alg->ValidateArguments())
     494           4 :             return false;
     495             :     }
     496             : 
     497         209 :     for (auto &step : steps)
     498         152 :         m_steps.push_back(std::move(step.alg));
     499             : 
     500          57 :     return true;
     501             : }
     502             : 
     503             : /************************************************************************/
     504             : /*            GDALRasterPipelineAlgorithm::GetUsageForCLI()             */
     505             : /************************************************************************/
     506             : 
     507           8 : std::string GDALRasterPipelineAlgorithm::GetUsageForCLI(
     508             :     bool shortUsage, const UsageOptions &usageOptions) const
     509             : {
     510           8 :     UsageOptions stepUsageOptions;
     511           8 :     stepUsageOptions.isPipelineStep = true;
     512             : 
     513           8 :     if (!m_helpDocCategory.empty() && m_helpDocCategory != "main")
     514             :     {
     515           4 :         auto alg = GetStepAlg(m_helpDocCategory);
     516           4 :         std::string ret;
     517           2 :         if (alg)
     518             :         {
     519           2 :             alg->SetCallPath({m_helpDocCategory});
     520           1 :             alg->GetArg("help-doc")->Set(true);
     521           1 :             return alg->GetUsageForCLI(shortUsage, stepUsageOptions);
     522             :         }
     523             :         else
     524             :         {
     525           1 :             fprintf(stderr, "ERROR: unknown pipeline step '%s'\n",
     526             :                     m_helpDocCategory.c_str());
     527             :             return CPLSPrintf("ERROR: unknown pipeline step '%s'\n",
     528           1 :                               m_helpDocCategory.c_str());
     529             :         }
     530             :     }
     531             : 
     532           6 :     UsageOptions usageOptionsMain(usageOptions);
     533           6 :     usageOptionsMain.isPipelineMain = true;
     534             :     std::string ret =
     535          12 :         GDALAlgorithm::GetUsageForCLI(shortUsage, usageOptionsMain);
     536           6 :     if (shortUsage)
     537           2 :         return ret;
     538             : 
     539             :     ret += "\n<PIPELINE> is of the form: read|mosaic|stack [READ-OPTIONS] "
     540           4 :            "( ! <STEP-NAME> [STEP-OPTIONS] )* ! write [WRITE-OPTIONS]\n";
     541             : 
     542           4 :     if (m_helpDocCategory == "main")
     543             :     {
     544           1 :         return ret;
     545             :     }
     546             : 
     547           3 :     ret += '\n';
     548           3 :     ret += "Example: 'gdal raster pipeline --progress ! read in.tif ! \\\n";
     549           3 :     ret += "               reproject --dst-crs=EPSG:32632 ! ";
     550           3 :     ret += "write out.tif --overwrite'\n";
     551           3 :     ret += '\n';
     552           3 :     ret += "Potential steps are:\n";
     553             : 
     554          90 :     for (const std::string &name : m_stepRegistry.GetNames())
     555             :     {
     556         174 :         auto alg = GetStepAlg(name);
     557          87 :         auto [options, maxOptLen] = alg->GetArgNamesForCLI();
     558          87 :         stepUsageOptions.maxOptLen =
     559          87 :             std::max(stepUsageOptions.maxOptLen, maxOptLen);
     560             :     }
     561             : 
     562             :     {
     563           3 :         const auto name = GDALRasterReadAlgorithm::NAME;
     564           3 :         ret += '\n';
     565           6 :         auto alg = GetStepAlg(name);
     566           6 :         alg->SetCallPath({name});
     567           3 :         ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
     568             :     }
     569          90 :     for (const std::string &name : m_stepRegistry.GetNames())
     570             :     {
     571         174 :         auto alg = GetStepAlg(name);
     572          87 :         assert(alg);
     573          99 :         if (alg->CanBeFirstStep() && !alg->IsHidden() &&
     574          12 :             name != GDALRasterReadAlgorithm::NAME)
     575             :         {
     576           9 :             ret += '\n';
     577          18 :             alg->SetCallPath({name});
     578           9 :             ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
     579             :         }
     580             :     }
     581          90 :     for (const std::string &name : m_stepRegistry.GetNames())
     582             :     {
     583         174 :         auto alg = GetStepAlg(name);
     584          87 :         assert(alg);
     585         162 :         if (!alg->CanBeFirstStep() && !alg->IsHidden() &&
     586          75 :             name != GDALRasterWriteAlgorithm::NAME)
     587             :         {
     588          72 :             ret += '\n';
     589         144 :             alg->SetCallPath({name});
     590          72 :             ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
     591             :         }
     592             :     }
     593             :     {
     594           3 :         const auto name = GDALRasterWriteAlgorithm::NAME;
     595           3 :         ret += '\n';
     596           6 :         auto alg = GetStepAlg(name);
     597           6 :         alg->SetCallPath({name});
     598           3 :         ret += alg->GetUsageForCLI(shortUsage, stepUsageOptions);
     599             :     }
     600             : 
     601           3 :     ret += GetUsageForCLIEnd();
     602             : 
     603           3 :     return ret;
     604             : }
     605             : 
     606             : /************************************************************************/
     607             : /*           GDALRasterPipelineNonNativelyStreamingAlgorithm()          */
     608             : /************************************************************************/
     609             : 
     610         189 : GDALRasterPipelineNonNativelyStreamingAlgorithm::
     611             :     GDALRasterPipelineNonNativelyStreamingAlgorithm(
     612             :         const std::string &name, const std::string &description,
     613         189 :         const std::string &helpURL, bool standaloneStep)
     614             :     : GDALRasterPipelineStepAlgorithm(name, description, helpURL,
     615         189 :                                       standaloneStep)
     616             : {
     617         189 : }
     618             : 
     619             : /************************************************************************/
     620             : /*                    IsNativelyStreamingCompatible()                   */
     621             : /************************************************************************/
     622             : 
     623          41 : bool GDALRasterPipelineNonNativelyStreamingAlgorithm::
     624             :     IsNativelyStreamingCompatible() const
     625             : {
     626          41 :     return false;
     627             : }
     628             : 
     629             : /************************************************************************/
     630             : /*                     MustCreateOnDiskTempDataset()                    */
     631             : /************************************************************************/
     632             : 
     633          55 : static bool MustCreateOnDiskTempDataset(int nWidth, int nHeight, int nBands,
     634             :                                         GDALDataType eDT)
     635             : {
     636             :     // Config option mostly for autotest purposes
     637          55 :     if (CPLTestBool(CPLGetConfigOption(
     638             :             "GDAL_RASTER_PIPELINE_USE_GTIFF_FOR_TEMP_DATASET", "NO")))
     639           5 :         return true;
     640             : 
     641             :     // Allow up to 10% of RAM usage for temporary dataset
     642          50 :     const auto nRAM = CPLGetUsablePhysicalRAM() / 10;
     643          50 :     const int nDTSize = GDALGetDataTypeSizeBytes(eDT);
     644          50 :     const bool bOnDisk =
     645         100 :         nBands > 0 && nDTSize > 0 && nRAM > 0 &&
     646          50 :         static_cast<int64_t>(nWidth) * nHeight > nRAM / (nBands * nDTSize);
     647          50 :     return bOnDisk;
     648             : }
     649             : 
     650             : /************************************************************************/
     651             : /*                      CreateTemporaryDataset()                        */
     652             : /************************************************************************/
     653             : 
     654             : std::unique_ptr<GDALDataset>
     655          30 : GDALRasterPipelineNonNativelyStreamingAlgorithm::CreateTemporaryDataset(
     656             :     int nWidth, int nHeight, int nBands, GDALDataType eDT,
     657             :     bool bTiledIfPossible, GDALDataset *poSrcDSForMetadata, bool bCopyMetadata)
     658             : {
     659             :     const bool bOnDisk =
     660          30 :         MustCreateOnDiskTempDataset(nWidth, nHeight, nBands, eDT);
     661          30 :     const char *pszDriverName = bOnDisk ? "GTIFF" : "MEM";
     662             :     GDALDriver *poDriver =
     663          30 :         GetGDALDriverManager()->GetDriverByName(pszDriverName);
     664          60 :     CPLStringList aosOptions;
     665          60 :     std::string osTmpFilename;
     666          30 :     if (bOnDisk)
     667             :     {
     668             :         osTmpFilename =
     669           4 :             CPLGenerateTempFilenameSafe(
     670             :                 poSrcDSForMetadata
     671           4 :                     ? CPLGetBasenameSafe(poSrcDSForMetadata->GetDescription())
     672           2 :                           .c_str()
     673           4 :                     : "") +
     674           2 :             ".tif";
     675           2 :         if (bTiledIfPossible)
     676           2 :             aosOptions.SetNameValue("TILED", "YES");
     677             :         const char *pszCOList =
     678           2 :             poDriver->GetMetadataItem(GDAL_DMD_CREATIONOPTIONLIST);
     679             :         aosOptions.SetNameValue("COMPRESS",
     680           2 :                                 pszCOList && strstr(pszCOList, "ZSTD") ? "ZSTD"
     681           4 :                                                                        : "LZW");
     682           2 :         aosOptions.SetNameValue("SPARSE_OK", "YES");
     683             :     }
     684             :     std::unique_ptr<GDALDataset> poOutDS(
     685          30 :         poDriver ? poDriver->Create(osTmpFilename.c_str(), nWidth, nHeight,
     686          30 :                                     nBands, eDT, aosOptions.List())
     687          60 :                  : nullptr);
     688          30 :     if (poOutDS && bOnDisk)
     689             :     {
     690             :         // In file systems that allow it (all but Windows...), we want to
     691             :         // delete the temporary file as soon as soon as possible after
     692             :         // having open it, so that if someone kills the process there are
     693             :         // no temp files left over. If that unlink() doesn't succeed
     694             :         // (on Windows), then the file will eventually be deleted when
     695             :         // poTmpDS is cleaned due to MarkSuppressOnClose().
     696           0 :         VSIUnlink(osTmpFilename.c_str());
     697           0 :         poOutDS->MarkSuppressOnClose();
     698             :     }
     699             : 
     700          30 :     if (poOutDS && poSrcDSForMetadata)
     701             :     {
     702          28 :         poOutDS->SetSpatialRef(poSrcDSForMetadata->GetSpatialRef());
     703          28 :         GDALGeoTransform gt;
     704          28 :         if (poSrcDSForMetadata->GetGeoTransform(gt) == CE_None)
     705          27 :             poOutDS->SetGeoTransform(gt);
     706          28 :         if (const int nGCPCount = poSrcDSForMetadata->GetGCPCount())
     707             :         {
     708           0 :             const auto apsGCPs = poSrcDSForMetadata->GetGCPs();
     709           0 :             if (apsGCPs)
     710             :             {
     711           0 :                 poOutDS->SetGCPs(nGCPCount, apsGCPs,
     712           0 :                                  poSrcDSForMetadata->GetGCPSpatialRef());
     713             :             }
     714             :         }
     715          28 :         if (bCopyMetadata)
     716             :         {
     717          14 :             poOutDS->SetMetadata(poSrcDSForMetadata->GetMetadata());
     718             :         }
     719             :     }
     720             : 
     721          60 :     return poOutDS;
     722             : }
     723             : 
     724             : /************************************************************************/
     725             : /*                       CreateTemporaryCopy()                          */
     726             : /************************************************************************/
     727             : 
     728             : std::unique_ptr<GDALDataset>
     729          25 : GDALRasterPipelineNonNativelyStreamingAlgorithm::CreateTemporaryCopy(
     730             :     GDALAlgorithm *poAlg, GDALDataset *poSrcDS, int nSingleBand,
     731             :     bool bTiledIfPossible, GDALProgressFunc pfnProgress, void *pProgressData)
     732             : {
     733          25 :     const int nBands = nSingleBand > 0 ? 1 : poSrcDS->GetRasterCount();
     734             :     const auto eDT =
     735          25 :         nBands ? poSrcDS->GetRasterBand(1)->GetRasterDataType() : GDT_Unknown;
     736          25 :     const bool bOnDisk = MustCreateOnDiskTempDataset(
     737             :         poSrcDS->GetRasterXSize(), poSrcDS->GetRasterYSize(), nBands, eDT);
     738          25 :     const char *pszDriverName = bOnDisk ? "GTIFF" : "MEM";
     739             : 
     740          50 :     CPLStringList options;
     741          25 :     if (nSingleBand > 0)
     742             :     {
     743          25 :         options.AddString("-b");
     744          25 :         options.AddString(CPLSPrintf("%d", nSingleBand));
     745             :     }
     746             : 
     747          25 :     options.AddString("-of");
     748          25 :     options.AddString(pszDriverName);
     749             : 
     750          50 :     std::string osTmpFilename;
     751          25 :     if (bOnDisk)
     752             :     {
     753             :         osTmpFilename =
     754           3 :             CPLGenerateTempFilenameSafe(
     755           9 :                 CPLGetBasenameSafe(poSrcDS->GetDescription()).c_str()) +
     756           3 :             ".tif";
     757           3 :         if (bTiledIfPossible)
     758             :         {
     759           3 :             options.AddString("-co");
     760           3 :             options.AddString("TILED=YES");
     761             :         }
     762             : 
     763             :         GDALDriver *poDriver =
     764           3 :             GetGDALDriverManager()->GetDriverByName(pszDriverName);
     765             :         const char *pszCOList =
     766           3 :             poDriver ? poDriver->GetMetadataItem(GDAL_DMD_CREATIONOPTIONLIST)
     767           3 :                      : nullptr;
     768           3 :         options.AddString("-co");
     769           3 :         options.AddString(pszCOList && strstr(pszCOList, "ZSTD")
     770             :                               ? "COMPRESS=ZSTD"
     771           6 :                               : "COMPRESS=LZW");
     772             :     }
     773             : 
     774             :     GDALTranslateOptions *translateOptions =
     775          25 :         GDALTranslateOptionsNew(options.List(), nullptr);
     776             : 
     777          25 :     if (pfnProgress)
     778          12 :         GDALTranslateOptionsSetProgress(translateOptions, pfnProgress,
     779             :                                         pProgressData);
     780             : 
     781             :     std::unique_ptr<GDALDataset> poOutDS(GDALDataset::FromHandle(
     782             :         GDALTranslate(osTmpFilename.c_str(), GDALDataset::ToHandle(poSrcDS),
     783          25 :                       translateOptions, nullptr)));
     784          25 :     GDALTranslateOptionsFree(translateOptions);
     785             : 
     786          25 :     if (!poOutDS)
     787             :     {
     788           2 :         poAlg->ReportError(CE_Failure, CPLE_AppDefined,
     789             :                            "Failed to create temporary dataset");
     790             :     }
     791          23 :     else if (bOnDisk)
     792             :     {
     793             :         // In file systems that allow it (all but Windows...), we want to
     794             :         // delete the temporary file as soon as soon as possible after
     795             :         // having open it, so that if someone kills the process there are
     796             :         // no temp files left over. If that unlink() doesn't succeed
     797             :         // (on Windows), then the file will eventually be deleted when
     798             :         // poTmpDS is cleaned due to MarkSuppressOnClose().
     799           1 :         VSIUnlink(osTmpFilename.c_str());
     800           1 :         poOutDS->MarkSuppressOnClose();
     801             :     }
     802          50 :     return poOutDS;
     803             : }
     804             : 
     805             : //! @endcond

Generated by: LCOV version 1.14